#pragma mark - Front Matter
#define _PTHREAD_CONDATTR_T
#define _PTHREAD_COND_T
#define _PTHREAD_MUTEXATTR_T
#define _PTHREAD_MUTEX_T
#define _PTHREAD_RWLOCKATTR_T
#define _PTHREAD_RWLOCK_T
#undef pthread_mutexattr_t
#undef pthread_mutex_t
#undef pthread_condattr_t
#undef pthread_cond_t
#undef pthread_rwlockattr_t
#undef pthread_rwlock_t
#include <sys/cdefs.h>
#include <os/log.h>
extern void panic(const char *string, ...) __printflike(1,2) __dead2;
#include <sys/param.h>
#include <sys/queue.h>
#include <sys/resourcevar.h>
#include <sys/kauth.h>
#include <sys/systm.h>
#include <sys/timeb.h>
#include <sys/times.h>
#include <sys/acct.h>
#include <sys/kernel.h>
#include <sys/wait.h>
#include <sys/signalvar.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/stat.h>
#include <sys/lock.h>
#include <sys/kdebug.h>
#include <sys/vm.h>
#include <sys/user.h>
#include <sys/proc_info.h>
#include <mach/mach_port.h>
#include <mach/mach_types.h>
#include <mach/semaphore.h>
#include <mach/sync_policy.h>
#include <mach/task.h>
#include <mach/vm_prot.h>
#include <kern/kern_types.h>
#include <kern/task.h>
#include <kern/clock.h>
#include <mach/kern_return.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <kern/sched_prim.h>
#include <kern/processor.h>
#include <kern/assert.h>
#include <mach/mach_vm.h>
#include <mach/mach_param.h>
#include <mach/thread_status.h>
#include <mach/thread_policy.h>
#include <mach/message.h>
#include <mach/port.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <mach/thread_act.h>
#include <machine/machine_routines.h>
#include <mach/shared_region.h>
#include <libkern/OSAtomic.h>
#include <libkern/libkern.h>
#include "kern_internal.h"
#ifndef WQ_SETUP_EXIT_THREAD
#define WQ_SETUP_EXIT_THREAD 8
#endif
#define THREAD_TAG_MAINTHREAD 0x1
#define THREAD_TAG_PTHREAD 0x10
#define THREAD_TAG_WORKQUEUE 0x20
lck_grp_attr_t *pthread_lck_grp_attr;
lck_grp_t *pthread_lck_grp;
lck_attr_t *pthread_lck_attr;
#define C_32_STK_ALIGN 16
#define C_64_STK_ALIGN 16
#define C_WORKQ_STK_ALIGN 16
#if defined(__arm64__)
#define PTHREAD_T_OFFSET (12*1024)
#else
#define PTHREAD_T_OFFSET 0
#endif
#define PTHREAD_START_CUSTOM 0x01000000 // <rdar://problem/34501401>
#define PTHREAD_START_SETSCHED 0x02000000
#define PTHREAD_START_QOSCLASS 0x08000000
#define PTHREAD_START_TSD_BASE_SET 0x10000000
#define PTHREAD_START_SUSPENDED 0x20000000
#define PTHREAD_START_QOSCLASS_MASK 0x00ffffff
#define PTHREAD_START_POLICY_BITSHIFT 16
#define PTHREAD_START_POLICY_MASK 0xff
#define PTHREAD_START_IMPORTANCE_MASK 0xffff
#define SCHED_OTHER POLICY_TIMESHARE
#define SCHED_FIFO POLICY_FIFO
#define SCHED_RR POLICY_RR
#define BASEPRI_DEFAULT 31
uint32_t pthread_debug_tracing = 1;
static uint32_t pthread_mutex_default_policy;
SYSCTL_INT(_kern, OID_AUTO, pthread_mutex_default_policy, CTLFLAG_RW | CTLFLAG_LOCKED,
&pthread_mutex_default_policy, 0, "");
#pragma mark - Process/Thread Setup/Teardown syscalls
static mach_vm_offset_t
stack_addr_hint(proc_t p, vm_map_t vmap)
{
mach_vm_offset_t stackaddr;
mach_vm_offset_t aslr_offset;
bool proc64bit = proc_is64bit(p);
bool proc64bit_data = proc_is64bit_data(p);
_Static_assert(powerof2(PTH_DEFAULT_STACKSIZE), "PTH_DEFAULT_STACKSIZE is a power-of-two");
#if defined(__i386__) || defined(__x86_64__)
(void)proc64bit_data;
if (proc64bit) {
aslr_offset = random() % (1 << 28); } else {
aslr_offset = random() % (16 * PTH_DEFAULT_STACKSIZE);
}
aslr_offset = vm_map_trunc_page_mask(aslr_offset, vm_map_page_mask(vmap));
if (proc64bit) {
stackaddr = 0x700000000000 + aslr_offset;
} else {
stackaddr = SHARED_REGION_BASE_I386 + SHARED_REGION_SIZE_I386 + aslr_offset;
}
#elif defined(__arm__) || defined(__arm64__)
user_addr_t main_thread_stack_top = 0;
if (pthread_kern->proc_get_user_stack) {
main_thread_stack_top = pthread_kern->proc_get_user_stack(p);
}
if (proc64bit && main_thread_stack_top) {
stackaddr = vm_map_trunc_page_mask((vm_map_offset_t)main_thread_stack_top,
vm_map_page_mask(vmap));
} else {
aslr_offset = random() % ((proc64bit ? 4 : 2) * PTH_DEFAULT_STACKSIZE);
aslr_offset = vm_map_trunc_page_mask((vm_map_offset_t)aslr_offset,
vm_map_page_mask(vmap));
if (proc64bit) {
stackaddr = SHARED_REGION_BASE_ARM64 - 64 * PTH_DEFAULT_STACKSIZE - aslr_offset;
} else {
if (proc64bit_data) {
stackaddr = SHARED_REGION_BASE_ARM64_32;
} else {
stackaddr = SHARED_REGION_BASE_ARM;
}
stackaddr -= 32 * PTH_DEFAULT_STACKSIZE + aslr_offset;
}
}
#else
#error Need to define a stack address hint for this architecture
#endif
return stackaddr;
}
static bool
_pthread_priority_to_policy(pthread_priority_t priority,
thread_qos_policy_data_t *data)
{
data->qos_tier = _pthread_priority_thread_qos(priority);
data->tier_importance = _pthread_priority_relpri(priority);
if (data->qos_tier == THREAD_QOS_UNSPECIFIED || data->tier_importance > 0 ||
data->tier_importance < THREAD_QOS_MIN_TIER_IMPORTANCE) {
return false;
}
return true;
}
int
_bsdthread_create(struct proc *p,
__unused user_addr_t user_func, __unused user_addr_t user_funcarg,
user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags,
user_addr_t *retval)
{
kern_return_t kret;
void * sright;
int error = 0;
mach_vm_offset_t th_tsd_base;
mach_port_name_t th_thport;
thread_t th;
task_t ctask = current_task();
unsigned int policy, importance;
uint32_t tsd_offset;
bool start_suspended = (flags & PTHREAD_START_SUSPENDED);
if (pthread_kern->proc_get_register(p) == 0) {
return EINVAL;
}
PTHREAD_TRACE(pthread_thread_create | DBG_FUNC_START, flags, 0, 0, 0);
kret = pthread_kern->thread_create(ctask, &th);
if (kret != KERN_SUCCESS)
return(ENOMEM);
thread_reference(th);
pthread_kern->thread_set_tag(th, THREAD_TAG_PTHREAD);
sright = (void *)pthread_kern->convert_thread_to_port(th);
th_thport = pthread_kern->ipc_port_copyout_send(sright, pthread_kern->task_get_ipcspace(ctask));
if (!MACH_PORT_VALID(th_thport)) {
error = EMFILE; goto out;
}
if ((flags & PTHREAD_START_CUSTOM) == 0) {
error = EINVAL;
goto out;
}
PTHREAD_TRACE(pthread_thread_create|DBG_FUNC_NONE, 0, 0, 0, 3);
tsd_offset = pthread_kern->proc_get_pthread_tsd_offset(p);
if (tsd_offset) {
th_tsd_base = user_pthread + tsd_offset;
kret = pthread_kern->thread_set_tsd_base(th, th_tsd_base);
if (kret == KERN_SUCCESS) {
flags |= PTHREAD_START_TSD_BASE_SET;
}
}
flags &= ~PTHREAD_START_SUSPENDED;
#if defined(__i386__) || defined(__x86_64__)
if (proc_is64bit_data(p)) {
x86_thread_state64_t state = {
.rip = (uint64_t)pthread_kern->proc_get_threadstart(p),
.rdi = (uint64_t)user_pthread,
.rsi = (uint64_t)th_thport,
.rdx = (uint64_t)user_func,
.rcx = (uint64_t)user_funcarg,
.r8 = (uint64_t)user_stack,
.r9 = (uint64_t)flags,
.rsp = (uint64_t)user_stack,
};
(void)pthread_kern->thread_set_wq_state64(th, (thread_state_t)&state);
} else {
x86_thread_state32_t state = {
.eip = (uint32_t)pthread_kern->proc_get_threadstart(p),
.eax = (uint32_t)user_pthread,
.ebx = (uint32_t)th_thport,
.ecx = (uint32_t)user_func,
.edx = (uint32_t)user_funcarg,
.edi = (uint32_t)user_stack,
.esi = (uint32_t)flags,
.esp = (uint32_t)user_stack,
};
(void)pthread_kern->thread_set_wq_state32(th, (thread_state_t)&state);
}
#elif defined(__arm__) || defined(__arm64__)
if (proc_is64bit_data(p)) {
#ifdef __arm64__
arm_thread_state64_t state = {
.pc = (uint64_t)pthread_kern->proc_get_threadstart(p),
.x[0] = (uint64_t)user_pthread,
.x[1] = (uint64_t)th_thport,
.x[2] = (uint64_t)user_func,
.x[3] = (uint64_t)user_funcarg,
.x[4] = (uint64_t)user_stack,
.x[5] = (uint64_t)flags,
.sp = (uint64_t)user_stack,
};
(void)pthread_kern->thread_set_wq_state64(th, (thread_state_t)&state);
#else
panic("Shouldn't have a 64-bit thread on a 32-bit kernel...");
#endif // defined(__arm64__)
} else {
arm_thread_state_t state = {
.pc = (uint32_t)pthread_kern->proc_get_threadstart(p),
.r[0] = (uint32_t)user_pthread,
.r[1] = (uint32_t)th_thport,
.r[2] = (uint32_t)user_func,
.r[3] = (uint32_t)user_funcarg,
.r[4] = (uint32_t)user_stack,
.r[5] = (uint32_t)flags,
.sp = (uint32_t)user_stack,
};
(void)pthread_kern->thread_set_wq_state32(th, (thread_state_t)&state);
}
#else
#error bsdthread_create not defined for this architecture
#endif
if (flags & PTHREAD_START_SETSCHED) {
thread_extended_policy_data_t extinfo;
thread_precedence_policy_data_t precedinfo;
importance = (flags & PTHREAD_START_IMPORTANCE_MASK);
policy = (flags >> PTHREAD_START_POLICY_BITSHIFT) & PTHREAD_START_POLICY_MASK;
if (policy == SCHED_OTHER) {
extinfo.timeshare = 1;
} else {
extinfo.timeshare = 0;
}
thread_policy_set(th, THREAD_EXTENDED_POLICY, (thread_policy_t)&extinfo, THREAD_EXTENDED_POLICY_COUNT);
precedinfo.importance = (importance - BASEPRI_DEFAULT);
thread_policy_set(th, THREAD_PRECEDENCE_POLICY, (thread_policy_t)&precedinfo, THREAD_PRECEDENCE_POLICY_COUNT);
} else if (flags & PTHREAD_START_QOSCLASS) {
thread_qos_policy_data_t qos;
if (!_pthread_priority_to_policy(flags & PTHREAD_START_QOSCLASS_MASK, &qos)) {
error = EINVAL;
goto out;
}
pthread_kern->thread_policy_set_internal(th, THREAD_QOS_POLICY,
(thread_policy_t)&qos, THREAD_QOS_POLICY_COUNT);
}
if (pthread_kern->proc_get_mach_thread_self_tsd_offset) {
uint64_t mach_thread_self_offset =
pthread_kern->proc_get_mach_thread_self_tsd_offset(p);
if (mach_thread_self_offset && tsd_offset) {
bool proc64bit = proc_is64bit(p);
if (proc64bit) {
uint64_t th_thport_tsd = (uint64_t)th_thport;
error = copyout(&th_thport_tsd, user_pthread + tsd_offset +
mach_thread_self_offset, sizeof(th_thport_tsd));
} else {
uint32_t th_thport_tsd = (uint32_t)th_thport;
error = copyout(&th_thport_tsd, user_pthread + tsd_offset +
mach_thread_self_offset, sizeof(th_thport_tsd));
}
if (error) {
goto out;
}
}
}
if (!start_suspended) {
kret = pthread_kern->thread_resume(th);
if (kret != KERN_SUCCESS) {
error = EINVAL;
goto out;
}
}
thread_deallocate(th);
PTHREAD_TRACE(pthread_thread_create|DBG_FUNC_END, error, user_pthread, 0, 0);
*retval = user_pthread;
return(0);
out:
(void)pthread_kern->mach_port_deallocate(pthread_kern->task_get_ipcspace(ctask), th_thport);
if (pthread_kern->thread_will_park_or_terminate) {
pthread_kern->thread_will_park_or_terminate(th);
}
(void)thread_terminate(th);
(void)thread_deallocate(th);
return(error);
}
int
_bsdthread_terminate(__unused struct proc *p,
user_addr_t stackaddr,
size_t size,
uint32_t kthport,
uint32_t sem,
__unused int32_t *retval)
{
mach_vm_offset_t freeaddr;
mach_vm_size_t freesize;
kern_return_t kret;
thread_t th = current_thread();
freeaddr = (mach_vm_offset_t)stackaddr;
freesize = size;
PTHREAD_TRACE(pthread_thread_terminate|DBG_FUNC_START, freeaddr, freesize, kthport, 0xff);
if ((freesize != (mach_vm_size_t)0) && (freeaddr != (mach_vm_offset_t)0)) {
if (pthread_kern->thread_get_tag(th) & THREAD_TAG_MAINTHREAD){
vm_map_t user_map = pthread_kern->current_map();
freesize = vm_map_trunc_page_mask((vm_map_offset_t)freesize - 1, vm_map_page_mask(user_map));
kret = mach_vm_behavior_set(user_map, freeaddr, freesize, VM_BEHAVIOR_REUSABLE);
#if MACH_ASSERT
if (kret != KERN_SUCCESS && kret != KERN_INVALID_ADDRESS) {
os_log_error(OS_LOG_DEFAULT, "unable to make thread stack reusable (kr: %d)", kret);
}
#endif
kret = kret ? kret : mach_vm_protect(user_map, freeaddr, freesize, FALSE, VM_PROT_NONE);
assert(kret == KERN_SUCCESS || kret == KERN_INVALID_ADDRESS);
} else {
kret = mach_vm_deallocate(pthread_kern->current_map(), freeaddr, freesize);
if (kret != KERN_SUCCESS) {
PTHREAD_TRACE(pthread_thread_terminate|DBG_FUNC_END, kret, 0, 0, 0);
}
}
}
if (pthread_kern->thread_will_park_or_terminate) {
pthread_kern->thread_will_park_or_terminate(th);
}
(void)thread_terminate(th);
if (sem != MACH_PORT_NULL) {
kret = pthread_kern->semaphore_signal_internal_trap(sem);
if (kret != KERN_SUCCESS) {
PTHREAD_TRACE(pthread_thread_terminate|DBG_FUNC_END, kret, 0, 0, 0);
}
}
if (kthport != MACH_PORT_NULL) {
pthread_kern->mach_port_deallocate(pthread_kern->task_get_ipcspace(current_task()), kthport);
}
PTHREAD_TRACE(pthread_thread_terminate|DBG_FUNC_END, 0, 0, 0, 0);
pthread_kern->thread_exception_return();
__builtin_unreachable();
}
int
_bsdthread_register(struct proc *p,
user_addr_t threadstart,
user_addr_t wqthread,
int pthsize,
user_addr_t pthread_init_data,
user_addr_t pthread_init_data_size,
uint64_t dispatchqueue_offset,
int32_t *retval)
{
struct _pthread_registration_data data = {};
uint32_t max_tsd_offset;
kern_return_t kr;
size_t pthread_init_sz = 0;
if (pthsize < 0 || pthsize > MAX_PTHREAD_SIZE) {
return(EINVAL);
}
if (pthread_init_data != 0) {
if (pthread_init_data_size < sizeof(data.version)) {
return EINVAL;
}
pthread_init_sz = MIN(sizeof(data), (size_t)pthread_init_data_size);
int ret = copyin(pthread_init_data, &data, pthread_init_sz);
if (ret) {
return ret;
}
if (data.version != (size_t)pthread_init_data_size) {
return EINVAL;
}
} else {
data.dispatch_queue_offset = dispatchqueue_offset;
}
mach_vm_offset_t stackaddr = stack_addr_hint(p, pthread_kern->current_map());
pthread_kern->proc_set_stack_addr_hint(p, (user_addr_t)stackaddr);
if (pthread_kern->proc_get_register(p) != 0) {
return(EINVAL);
}
pthread_kern->proc_set_threadstart(p, threadstart);
pthread_kern->proc_set_wqthread(p, wqthread);
pthread_kern->proc_set_pthsize(p, pthsize);
pthread_kern->proc_set_register(p);
uint32_t tsd_slot_sz = proc_is64bit(p) ? sizeof(uint64_t) : sizeof(uint32_t);
if ((uint32_t)pthsize >= tsd_slot_sz &&
data.tsd_offset <= (uint32_t)(pthsize - tsd_slot_sz)) {
max_tsd_offset = ((uint32_t)pthsize - data.tsd_offset - tsd_slot_sz);
} else {
data.tsd_offset = 0;
max_tsd_offset = 0;
}
pthread_kern->proc_set_pthread_tsd_offset(p, data.tsd_offset);
if (data.dispatch_queue_offset > max_tsd_offset) {
data.dispatch_queue_offset = 0;
}
pthread_kern->proc_set_dispatchqueue_offset(p, data.dispatch_queue_offset);
if (pthread_kern->proc_set_return_to_kernel_offset) {
if (data.return_to_kernel_offset > max_tsd_offset) {
data.return_to_kernel_offset = 0;
}
pthread_kern->proc_set_return_to_kernel_offset(p,
data.return_to_kernel_offset);
}
if (pthread_kern->proc_set_mach_thread_self_tsd_offset) {
if (data.mach_thread_self_offset > max_tsd_offset) {
data.mach_thread_self_offset = 0;
}
pthread_kern->proc_set_mach_thread_self_tsd_offset(p,
data.mach_thread_self_offset);
}
if (pthread_init_data != 0) {
data.version = sizeof(struct _pthread_registration_data);
data.main_qos = _pthread_unspecified_priority();
if (pthread_kern->qos_main_thread_active()) {
mach_msg_type_number_t nqos = THREAD_QOS_POLICY_COUNT;
thread_qos_policy_data_t qos;
boolean_t gd = FALSE;
kr = pthread_kern->thread_policy_get(current_thread(),
THREAD_QOS_POLICY, (thread_policy_t)&qos, &nqos, &gd);
if (kr != KERN_SUCCESS || qos.qos_tier == THREAD_QOS_UNSPECIFIED) {
qos.qos_tier = THREAD_QOS_LEGACY;
qos.tier_importance = 0;
kr = pthread_kern->thread_policy_set_internal(current_thread(),
THREAD_QOS_POLICY, (thread_policy_t)&qos,
THREAD_QOS_POLICY_COUNT);
}
if (kr == KERN_SUCCESS) {
data.main_qos = _pthread_priority_make_from_thread_qos(
qos.qos_tier, 0, 0);
}
}
data.stack_addr_hint = stackaddr;
data.mutex_default_policy = pthread_mutex_default_policy;
kr = copyout(&data, pthread_init_data, pthread_init_sz);
if (kr != KERN_SUCCESS) {
return EINVAL;
}
}
*retval = PTHREAD_FEATURE_SUPPORTED;
return(0);
}
#pragma mark - Workqueue Thread Support
static mach_vm_size_t
workq_thread_allocsize(proc_t p, vm_map_t wq_map,
mach_vm_size_t *guardsize_out)
{
mach_vm_size_t guardsize = vm_map_page_size(wq_map);
mach_vm_size_t pthread_size = vm_map_round_page_mask(
pthread_kern->proc_get_pthsize(p) + PTHREAD_T_OFFSET,
vm_map_page_mask(wq_map));
if (guardsize_out) *guardsize_out = guardsize;
return guardsize + PTH_DEFAULT_STACKSIZE + pthread_size;
}
int
workq_create_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t *out_addr)
{
mach_vm_offset_t stackaddr = pthread_kern->proc_get_stack_addr_hint(p);
mach_vm_size_t guardsize, th_allocsize;
kern_return_t kret;
th_allocsize = workq_thread_allocsize(p, vmap, &guardsize);
kret = mach_vm_map(vmap, &stackaddr, th_allocsize, page_size - 1,
VM_MAKE_TAG(VM_MEMORY_STACK) | VM_FLAGS_ANYWHERE, NULL, 0, FALSE,
VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if (kret != KERN_SUCCESS) {
kret = mach_vm_allocate(vmap, &stackaddr, th_allocsize,
VM_MAKE_TAG(VM_MEMORY_STACK) | VM_FLAGS_ANYWHERE);
}
if (kret != KERN_SUCCESS) {
goto fail;
}
kret = mach_vm_protect(vmap, stackaddr, guardsize, FALSE, VM_PROT_NONE);
if (kret != KERN_SUCCESS) {
goto fail_vm_deallocate;
}
if (out_addr) {
*out_addr = stackaddr;
}
return 0;
fail_vm_deallocate:
(void)mach_vm_deallocate(vmap, stackaddr, th_allocsize);
fail:
return kret;
}
int
workq_destroy_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t stackaddr)
{
return mach_vm_deallocate(vmap, stackaddr,
workq_thread_allocsize(p, vmap, NULL));
}
void
workq_markfree_threadstack(proc_t OS_UNUSED p, thread_t OS_UNUSED th,
vm_map_t vmap, user_addr_t stackaddr)
{
const vm_size_t guardsize = vm_map_page_size(vmap);
const user_addr_t freeaddr = (user_addr_t)stackaddr + guardsize;
const vm_map_offset_t freesize = vm_map_trunc_page_mask(
(PTH_DEFAULT_STACKSIZE + guardsize + PTHREAD_T_OFFSET) - 1,
vm_map_page_mask(vmap)) - guardsize;
__assert_only kern_return_t kr = mach_vm_behavior_set(vmap, freeaddr,
freesize, VM_BEHAVIOR_REUSABLE);
#if MACH_ASSERT
if (kr != KERN_SUCCESS && kr != KERN_INVALID_ADDRESS) {
os_log_error(OS_LOG_DEFAULT, "unable to make thread stack reusable (kr: %d)", kr);
}
#endif
}
struct workq_thread_addrs {
user_addr_t self;
user_addr_t stack_bottom;
user_addr_t stack_top;
};
static inline void
workq_thread_set_top_addr(struct workq_thread_addrs *th_addrs, user_addr_t addr)
{
th_addrs->stack_top = (addr & -C_WORKQ_STK_ALIGN);
}
static void
workq_thread_get_addrs(vm_map_t map, user_addr_t stackaddr,
struct workq_thread_addrs *th_addrs)
{
const vm_size_t guardsize = vm_map_page_size(map);
th_addrs->self = (user_addr_t)(stackaddr + PTH_DEFAULT_STACKSIZE +
guardsize + PTHREAD_T_OFFSET);
workq_thread_set_top_addr(th_addrs, th_addrs->self);
th_addrs->stack_bottom = (user_addr_t)(stackaddr + guardsize);
}
static inline void
workq_set_register_state(proc_t p, thread_t th,
struct workq_thread_addrs *addrs, mach_port_name_t kport,
user_addr_t kevent_list, uint32_t upcall_flags, int kevent_count)
{
user_addr_t wqstart_fnptr = pthread_kern->proc_get_wqthread(p);
if (!wqstart_fnptr) {
panic("workqueue thread start function pointer is NULL");
}
#if defined(__i386__) || defined(__x86_64__)
if (proc_is64bit_data(p) == 0) {
x86_thread_state32_t state = {
.eip = (unsigned int)wqstart_fnptr,
.eax = (unsigned int)addrs->self,
.ebx = (unsigned int)kport,
.ecx = (unsigned int)addrs->stack_bottom,
.edx = (unsigned int)kevent_list,
.edi = (unsigned int)upcall_flags,
.esi = (unsigned int)kevent_count,
.esp = (int)((vm_offset_t)addrs->stack_top),
};
int error = pthread_kern->thread_set_wq_state32(th, (thread_state_t)&state);
if (error != KERN_SUCCESS) {
panic(__func__ ": thread_set_wq_state failed: %d", error);
}
} else {
x86_thread_state64_t state64 = {
.rip = (uint64_t)wqstart_fnptr,
.rdi = (uint64_t)addrs->self,
.rsi = (uint64_t)kport,
.rdx = (uint64_t)addrs->stack_bottom,
.rcx = (uint64_t)kevent_list,
.r8 = (uint64_t)upcall_flags,
.r9 = (uint64_t)kevent_count,
.rsp = (uint64_t)(addrs->stack_top)
};
int error = pthread_kern->thread_set_wq_state64(th, (thread_state_t)&state64);
if (error != KERN_SUCCESS) {
panic(__func__ ": thread_set_wq_state failed: %d", error);
}
}
#elif defined(__arm__) || defined(__arm64__)
if (!proc_is64bit_data(p)) {
arm_thread_state_t state = {
.pc = (int)wqstart_fnptr,
.r[0] = (unsigned int)addrs->self,
.r[1] = (unsigned int)kport,
.r[2] = (unsigned int)addrs->stack_bottom,
.r[3] = (unsigned int)kevent_list,
.r[4] = (unsigned int)upcall_flags,
.r[5] = (unsigned int)kevent_count,
.sp = (int)(addrs->stack_top)
};
int error = pthread_kern->thread_set_wq_state32(th, (thread_state_t)&state);
if (error != KERN_SUCCESS) {
panic(__func__ ": thread_set_wq_state failed: %d", error);
}
} else {
#if defined(__arm64__)
arm_thread_state64_t state = {
.pc = (uint64_t)wqstart_fnptr,
.x[0] = (uint64_t)addrs->self,
.x[1] = (uint64_t)kport,
.x[2] = (uint64_t)addrs->stack_bottom,
.x[3] = (uint64_t)kevent_list,
.x[4] = (uint64_t)upcall_flags,
.x[5] = (uint64_t)kevent_count,
.sp = (uint64_t)((vm_offset_t)addrs->stack_top),
};
int error = pthread_kern->thread_set_wq_state64(th, (thread_state_t)&state);
if (error != KERN_SUCCESS) {
panic(__func__ ": thread_set_wq_state failed: %d", error);
}
#else
panic("Shouldn't have a 64-bit thread on a 32-bit kernel...");
#endif
}
#else
#error setup_wqthread not defined for this architecture
#endif
}
static inline int
workq_kevent(proc_t p, struct workq_thread_addrs *th_addrs,
user_addr_t eventlist, int nevents, int kevent_flags,
user_addr_t *kevent_list_out, int *kevent_count_out)
{
int ret;
user_addr_t kevent_list = th_addrs->self -
WQ_KEVENT_LIST_LEN * sizeof(struct kevent_qos_s);
user_addr_t data_buf = kevent_list - WQ_KEVENT_DATA_SIZE;
user_size_t data_available = WQ_KEVENT_DATA_SIZE;
ret = pthread_kern->kevent_workq_internal(p, eventlist, nevents,
kevent_list, WQ_KEVENT_LIST_LEN,
data_buf, &data_available,
kevent_flags, kevent_count_out);
if (ret != 0 || *kevent_count_out == -1) {
*kevent_list_out = NULL;
*kevent_count_out = 0;
return ret;
}
workq_thread_set_top_addr(th_addrs, data_buf + data_available);
*kevent_list_out = kevent_list;
return ret;
}
__attribute__((noreturn,noinline))
void
workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr,
mach_port_name_t kport, int th_qos __unused, int setup_flags, int upcall_flags)
{
struct workq_thread_addrs th_addrs;
bool first_use = (setup_flags & WQ_SETUP_FIRST_USE);
user_addr_t kevent_list = NULL;
int kevent_count = 0;
workq_thread_get_addrs(map, stackaddr, &th_addrs);
if (first_use) {
uint32_t tsd_offset = pthread_kern->proc_get_pthread_tsd_offset(p);
if (tsd_offset) {
mach_vm_offset_t th_tsd_base = th_addrs.self + tsd_offset;
kern_return_t kret = pthread_kern->thread_set_tsd_base(th,
th_tsd_base);
if (kret == KERN_SUCCESS) {
upcall_flags |= WQ_FLAG_THREAD_TSD_BASE_SET;
}
}
vm_map_offset_t mask = vm_map_page_mask(map);
vm_map_offset_t th_page = vm_map_trunc_page_mask(th_addrs.self, mask);
vm_map_offset_t stk_page = vm_map_trunc_page_mask(th_addrs.stack_top - 1, mask);
if (th_page != stk_page) {
vm_fault(map, stk_page, VM_PROT_READ | VM_PROT_WRITE, FALSE, THREAD_UNINT, NULL, 0);
}
vm_fault(map, th_page, VM_PROT_READ | VM_PROT_WRITE, FALSE, THREAD_UNINT, NULL, 0);
}
if (setup_flags & WQ_SETUP_EXIT_THREAD) {
kevent_count = WORKQ_EXIT_THREAD_NKEVENT;
} else if (upcall_flags & WQ_FLAG_THREAD_KEVENT) {
unsigned int flags = KEVENT_FLAG_STACK_DATA | KEVENT_FLAG_IMMEDIATE;
workq_kevent(p, &th_addrs, NULL, 0, flags, &kevent_list, &kevent_count);
}
workq_set_register_state(p, th, &th_addrs, kport,
kevent_list, upcall_flags, kevent_count);
if (first_use) {
pthread_kern->thread_bootstrap_return();
} else {
pthread_kern->unix_syscall_return(EJUSTRETURN);
}
__builtin_unreachable();
}
int
workq_handle_stack_events(proc_t p, thread_t th, vm_map_t map,
user_addr_t stackaddr, mach_port_name_t kport,
user_addr_t events, int nevents, int upcall_flags)
{
struct workq_thread_addrs th_addrs;
user_addr_t kevent_list = NULL;
int kevent_count = 0, error;
__assert_only kern_return_t kr;
workq_thread_get_addrs(map, stackaddr, &th_addrs);
unsigned int flags = KEVENT_FLAG_STACK_DATA | KEVENT_FLAG_IMMEDIATE |
KEVENT_FLAG_PARKING;
error = workq_kevent(p, &th_addrs, events, nevents, flags,
&kevent_list, &kevent_count);
if (error || kevent_count == 0) {
return error;
}
kr = pthread_kern->thread_set_voucher_name(MACH_PORT_NULL);
assert(kr == KERN_SUCCESS);
workq_set_register_state(p, th, &th_addrs, kport,
kevent_list, upcall_flags, kevent_count);
pthread_kern->unix_syscall_return(EJUSTRETURN);
__builtin_unreachable();
}
int
_thread_selfid(__unused struct proc *p, uint64_t *retval)
{
thread_t thread = current_thread();
*retval = thread_tid(thread);
return KERN_SUCCESS;
}
void
_pthread_init(void)
{
pthread_lck_grp_attr = lck_grp_attr_alloc_init();
pthread_lck_grp = lck_grp_alloc_init("pthread", pthread_lck_grp_attr);
pthread_lck_attr = lck_attr_alloc_init();
pthread_list_mlock = lck_mtx_alloc_init(pthread_lck_grp, pthread_lck_attr);
pth_global_hashinit();
psynch_thcall = thread_call_allocate(psynch_wq_cleanup, NULL);
psynch_zoneinit();
int policy_bootarg;
if (PE_parse_boot_argn("pthread_mutex_default_policy", &policy_bootarg, sizeof(policy_bootarg))) {
pthread_mutex_default_policy = policy_bootarg;
}
sysctl_register_oid(&sysctl__kern_pthread_mutex_default_policy);
}