#include <mach/mach_types.h>
#include <mach/mach_traps.h>
#include <mach/kern_return.h>
#include <mach/semaphore.h>
#include <mach/sync_policy.h>
#include <mach/task.h>
#include <kern/misc_protos.h>
#include <kern/sync_sema.h>
#include <kern/spl.h>
#include <kern/ipc_kobject.h>
#include <kern/ipc_sync.h>
#include <kern/ipc_tt.h>
#include <kern/thread.h>
#include <kern/clock.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_space.h>
#include <kern/host.h>
#include <kern/waitq.h>
#include <kern/zalloc.h>
#include <kern/mach_param.h>
#include <libkern/OSAtomic.h>
static unsigned int semaphore_event;
#define SEMAPHORE_EVENT CAST_EVENT64_T(&semaphore_event)
ZONE_DECLARE(semaphore_zone, "semaphores", sizeof(struct semaphore), ZC_NONE);
os_refgrp_decl(static, sema_refgrp, "semaphore", NULL);
kern_return_t
semaphore_wait_trap_internal(
mach_port_name_t name,
void (*caller_cont)(kern_return_t));
kern_return_t
semaphore_wait_signal_trap_internal(
mach_port_name_t wait_name,
mach_port_name_t signal_name,
void (*caller_cont)(kern_return_t));
kern_return_t
semaphore_timedwait_trap_internal(
mach_port_name_t name,
unsigned int sec,
clock_res_t nsec,
void (*caller_cont)(kern_return_t));
kern_return_t
semaphore_timedwait_signal_trap_internal(
mach_port_name_t wait_name,
mach_port_name_t signal_name,
unsigned int sec,
clock_res_t nsec,
void (*caller_cont)(kern_return_t));
kern_return_t
semaphore_signal_internal_trap(mach_port_name_t sema_name);
kern_return_t
semaphore_signal_internal(
semaphore_t semaphore,
thread_t thread,
int options);
kern_return_t
semaphore_convert_wait_result(
int wait_result);
void
semaphore_wait_continue(void *arg __unused, wait_result_t wr);
static kern_return_t
semaphore_wait_internal(
semaphore_t wait_semaphore,
semaphore_t signal_semaphore,
uint64_t deadline,
int option,
void (*caller_cont)(kern_return_t));
static __inline__ uint64_t
semaphore_deadline(
unsigned int sec,
clock_res_t nsec)
{
uint64_t abstime;
nanoseconds_to_absolutetime((uint64_t)sec * NSEC_PER_SEC + nsec, &abstime);
clock_absolutetime_interval_to_deadline(abstime, &abstime);
return abstime;
}
kern_return_t
semaphore_create(
task_t task,
semaphore_t *new_semaphore,
int policy,
int value)
{
semaphore_t s = SEMAPHORE_NULL;
kern_return_t kret;
*new_semaphore = SEMAPHORE_NULL;
if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX || policy < 0) {
return KERN_INVALID_ARGUMENT;
}
s = (semaphore_t) zalloc(semaphore_zone);
if (s == SEMAPHORE_NULL) {
return KERN_RESOURCE_SHORTAGE;
}
kret = waitq_init(&s->waitq, policy | SYNC_POLICY_DISABLE_IRQ);
if (kret != KERN_SUCCESS) {
zfree(semaphore_zone, s);
return kret;
}
s->port = IP_NULL;
os_ref_init(&s->ref_count, &sema_refgrp);
s->count = value;
s->active = TRUE;
s->owner = task;
task_lock(task);
if (!task->active) {
task_unlock(task);
zfree(semaphore_zone, s);
return KERN_INVALID_TASK;
}
enqueue_head(&task->semaphore_list, (queue_entry_t) s);
task->semaphores_owned++;
task_unlock(task);
*new_semaphore = s;
return KERN_SUCCESS;
}
static void
semaphore_destroy_internal(
task_t task,
semaphore_t semaphore)
{
int old_count;
assert(semaphore->owner == task);
remqueue((queue_entry_t) semaphore);
semaphore->owner = TASK_NULL;
task->semaphores_owned--;
assert(semaphore->active);
semaphore->active = FALSE;
old_count = semaphore->count;
semaphore->count = 0;
if (old_count < 0) {
waitq_wakeup64_all_locked(&semaphore->waitq,
SEMAPHORE_EVENT,
THREAD_RESTART, NULL,
WAITQ_ALL_PRIORITIES,
WAITQ_UNLOCK);
} else {
semaphore_unlock(semaphore);
}
}
kern_return_t
semaphore_destroy(
task_t task,
semaphore_t semaphore)
{
spl_t spl_level;
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
if (task == TASK_NULL) {
semaphore_dereference(semaphore);
return KERN_INVALID_ARGUMENT;
}
task_lock(task);
spl_level = splsched();
semaphore_lock(semaphore);
if (semaphore->owner != task) {
semaphore_unlock(semaphore);
semaphore_dereference(semaphore);
splx(spl_level);
task_unlock(task);
return KERN_INVALID_ARGUMENT;
}
semaphore_destroy_internal(task, semaphore);
splx(spl_level);
task_unlock(task);
semaphore_dereference(semaphore);
return KERN_SUCCESS;
}
#define SEMASPERSPL 20
void
semaphore_destroy_all(
task_t task)
{
uint32_t count;
spl_t spl_level;
count = 0;
task_lock(task);
while (!queue_empty(&task->semaphore_list)) {
semaphore_t semaphore;
semaphore = (semaphore_t) queue_first(&task->semaphore_list);
if (count == 0) {
spl_level = splsched();
}
semaphore_lock(semaphore);
semaphore_destroy_internal(task, semaphore);
if (++count == SEMASPERSPL) {
count = 0;
splx(spl_level);
}
}
if (count != 0) {
splx(spl_level);
}
task_unlock(task);
}
kern_return_t
semaphore_signal_internal(
semaphore_t semaphore,
thread_t thread,
int options)
{
kern_return_t kr;
spl_t spl_level;
spl_level = splsched();
semaphore_lock(semaphore);
if (!semaphore->active) {
semaphore_unlock(semaphore);
splx(spl_level);
return KERN_TERMINATED;
}
if (thread != THREAD_NULL) {
if (semaphore->count < 0) {
kr = waitq_wakeup64_thread_locked(
&semaphore->waitq,
SEMAPHORE_EVENT,
thread,
THREAD_AWAKENED,
WAITQ_UNLOCK);
} else {
kr = KERN_NOT_WAITING;
semaphore_unlock(semaphore);
}
splx(spl_level);
return kr;
}
if (options & SEMAPHORE_SIGNAL_ALL) {
int old_count = semaphore->count;
kr = KERN_NOT_WAITING;
if (old_count < 0) {
semaphore->count = 0;
kr = waitq_wakeup64_all_locked(
&semaphore->waitq,
SEMAPHORE_EVENT,
THREAD_AWAKENED, NULL,
WAITQ_ALL_PRIORITIES,
WAITQ_UNLOCK);
} else {
if (options & SEMAPHORE_SIGNAL_PREPOST) {
semaphore->count++;
}
kr = KERN_SUCCESS;
semaphore_unlock(semaphore);
}
splx(spl_level);
return kr;
}
if (semaphore->count < 0) {
waitq_options_t wq_option = (options & SEMAPHORE_THREAD_HANDOFF) ?
WQ_OPTION_HANDOFF : WQ_OPTION_NONE;
kr = waitq_wakeup64_one_locked(
&semaphore->waitq,
SEMAPHORE_EVENT,
THREAD_AWAKENED, NULL,
WAITQ_ALL_PRIORITIES,
WAITQ_KEEP_LOCKED,
wq_option);
if (kr == KERN_SUCCESS) {
semaphore_unlock(semaphore);
splx(spl_level);
return KERN_SUCCESS;
} else {
semaphore->count = 0;
}
}
if (options & SEMAPHORE_SIGNAL_PREPOST) {
semaphore->count++;
}
semaphore_unlock(semaphore);
splx(spl_level);
return KERN_NOT_WAITING;
}
kern_return_t
semaphore_signal_thread(
semaphore_t semaphore,
thread_t thread)
{
kern_return_t ret;
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
ret = semaphore_signal_internal(semaphore,
thread,
SEMAPHORE_OPTION_NONE);
return ret;
}
kern_return_t
semaphore_signal_thread_trap(
struct semaphore_signal_thread_trap_args *args)
{
mach_port_name_t sema_name = args->signal_name;
mach_port_name_t thread_name = args->thread_name;
semaphore_t semaphore;
thread_t thread;
kern_return_t kr;
if (thread_name != MACH_PORT_NULL) {
thread = port_name_to_thread(thread_name, PORT_TO_THREAD_NONE);
if (thread == THREAD_NULL) {
return KERN_INVALID_ARGUMENT;
}
} else {
thread = THREAD_NULL;
}
kr = port_name_to_semaphore(sema_name, &semaphore);
if (kr == KERN_SUCCESS) {
kr = semaphore_signal_internal(semaphore,
thread,
SEMAPHORE_OPTION_NONE);
semaphore_dereference(semaphore);
}
if (thread != THREAD_NULL) {
thread_deallocate(thread);
}
return kr;
}
kern_return_t
semaphore_signal(
semaphore_t semaphore)
{
kern_return_t kr;
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
kr = semaphore_signal_internal(semaphore,
THREAD_NULL,
SEMAPHORE_SIGNAL_PREPOST);
if (kr == KERN_NOT_WAITING) {
return KERN_SUCCESS;
}
return kr;
}
kern_return_t
semaphore_signal_trap(
struct semaphore_signal_trap_args *args)
{
mach_port_name_t sema_name = args->signal_name;
return semaphore_signal_internal_trap(sema_name);
}
kern_return_t
semaphore_signal_internal_trap(mach_port_name_t sema_name)
{
semaphore_t semaphore;
kern_return_t kr;
kr = port_name_to_semaphore(sema_name, &semaphore);
if (kr == KERN_SUCCESS) {
kr = semaphore_signal_internal(semaphore,
THREAD_NULL,
SEMAPHORE_SIGNAL_PREPOST);
semaphore_dereference(semaphore);
if (kr == KERN_NOT_WAITING) {
kr = KERN_SUCCESS;
}
}
return kr;
}
kern_return_t
semaphore_signal_all(
semaphore_t semaphore)
{
kern_return_t kr;
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
kr = semaphore_signal_internal(semaphore,
THREAD_NULL,
SEMAPHORE_SIGNAL_ALL);
if (kr == KERN_NOT_WAITING) {
return KERN_SUCCESS;
}
return kr;
}
kern_return_t
semaphore_signal_all_trap(
struct semaphore_signal_all_trap_args *args)
{
mach_port_name_t sema_name = args->signal_name;
semaphore_t semaphore;
kern_return_t kr;
kr = port_name_to_semaphore(sema_name, &semaphore);
if (kr == KERN_SUCCESS) {
kr = semaphore_signal_internal(semaphore,
THREAD_NULL,
SEMAPHORE_SIGNAL_ALL);
semaphore_dereference(semaphore);
if (kr == KERN_NOT_WAITING) {
kr = KERN_SUCCESS;
}
}
return kr;
}
kern_return_t
semaphore_convert_wait_result(int wait_result)
{
switch (wait_result) {
case THREAD_AWAKENED:
return KERN_SUCCESS;
case THREAD_TIMED_OUT:
return KERN_OPERATION_TIMED_OUT;
case THREAD_INTERRUPTED:
return KERN_ABORTED;
case THREAD_RESTART:
return KERN_TERMINATED;
default:
panic("semaphore_block\n");
return KERN_FAILURE;
}
}
void
semaphore_wait_continue(void *arg __unused, wait_result_t wr)
{
thread_t self = current_thread();
void (*caller_cont)(kern_return_t) = self->sth_continuation;
assert(self->sth_waitsemaphore != SEMAPHORE_NULL);
semaphore_dereference(self->sth_waitsemaphore);
if (self->sth_signalsemaphore != SEMAPHORE_NULL) {
semaphore_dereference(self->sth_signalsemaphore);
}
assert(self->handoff_thread == THREAD_NULL);
assert(caller_cont != (void (*)(kern_return_t))0);
(*caller_cont)(semaphore_convert_wait_result(wr));
}
static kern_return_t
semaphore_wait_internal(
semaphore_t wait_semaphore,
semaphore_t signal_semaphore,
uint64_t deadline,
int option,
void (*caller_cont)(kern_return_t))
{
int wait_result;
spl_t spl_level;
kern_return_t kr = KERN_ALREADY_WAITING;
spl_level = splsched();
semaphore_lock(wait_semaphore);
thread_t self = current_thread();
thread_t handoff_thread = THREAD_NULL;
thread_handoff_option_t handoff_option = THREAD_HANDOFF_NONE;
int semaphore_signal_options = SEMAPHORE_SIGNAL_PREPOST;
if (!wait_semaphore->active) {
kr = KERN_TERMINATED;
} else if (wait_semaphore->count > 0) {
wait_semaphore->count--;
kr = KERN_SUCCESS;
} else if (option & SEMAPHORE_TIMEOUT_NOBLOCK) {
kr = KERN_OPERATION_TIMED_OUT;
} else {
wait_semaphore->count = -1;
thread_set_pending_block_hint(self, kThreadWaitSemaphore);
(void)waitq_assert_wait64_locked(
&wait_semaphore->waitq,
SEMAPHORE_EVENT,
THREAD_ABORTSAFE,
TIMEOUT_URGENCY_USER_NORMAL,
deadline, TIMEOUT_NO_LEEWAY,
self);
semaphore_signal_options |= SEMAPHORE_THREAD_HANDOFF;
}
semaphore_unlock(wait_semaphore);
splx(spl_level);
if (signal_semaphore != SEMAPHORE_NULL) {
kern_return_t signal_kr;
signal_kr = semaphore_signal_internal(signal_semaphore,
THREAD_NULL, semaphore_signal_options);
if (signal_kr == KERN_NOT_WAITING) {
assert(self->handoff_thread == THREAD_NULL);
signal_kr = KERN_SUCCESS;
} else if (signal_kr == KERN_TERMINATED) {
assert(self->handoff_thread == THREAD_NULL);
clear_wait(self, THREAD_INTERRUPTED);
kr = semaphore_convert_wait_result(self->wait_result);
if (kr == KERN_ABORTED) {
kr = KERN_TERMINATED;
}
}
}
if (kr != KERN_ALREADY_WAITING) {
assert(self->handoff_thread == THREAD_NULL);
return kr;
}
if (self->handoff_thread) {
handoff_thread = self->handoff_thread;
self->handoff_thread = THREAD_NULL;
handoff_option = THREAD_HANDOFF_SETRUN_NEEDED;
}
if (caller_cont) {
self->sth_continuation = caller_cont;
self->sth_waitsemaphore = wait_semaphore;
self->sth_signalsemaphore = signal_semaphore;
thread_handoff_parameter(handoff_thread, semaphore_wait_continue,
NULL, handoff_option);
} else {
wait_result = thread_handoff_deallocate(handoff_thread, handoff_option);
}
assert(self->handoff_thread == THREAD_NULL);
return semaphore_convert_wait_result(wait_result);
}
kern_return_t
semaphore_wait(
semaphore_t semaphore)
{
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
return semaphore_wait_internal(semaphore,
SEMAPHORE_NULL,
0ULL, SEMAPHORE_OPTION_NONE,
(void (*)(kern_return_t))0);
}
kern_return_t
semaphore_wait_noblock(
semaphore_t semaphore)
{
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
return semaphore_wait_internal(semaphore,
SEMAPHORE_NULL,
0ULL, SEMAPHORE_TIMEOUT_NOBLOCK,
(void (*)(kern_return_t))0);
}
kern_return_t
semaphore_wait_deadline(
semaphore_t semaphore,
uint64_t deadline)
{
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
return semaphore_wait_internal(semaphore,
SEMAPHORE_NULL,
deadline, SEMAPHORE_OPTION_NONE,
(void (*)(kern_return_t))0);
}
kern_return_t
semaphore_wait_trap(
struct semaphore_wait_trap_args *args)
{
return semaphore_wait_trap_internal(args->wait_name, thread_syscall_return);
}
kern_return_t
semaphore_wait_trap_internal(
mach_port_name_t name,
void (*caller_cont)(kern_return_t))
{
semaphore_t semaphore;
kern_return_t kr;
kr = port_name_to_semaphore(name, &semaphore);
if (kr == KERN_SUCCESS) {
kr = semaphore_wait_internal(semaphore,
SEMAPHORE_NULL,
0ULL, SEMAPHORE_OPTION_NONE,
caller_cont);
semaphore_dereference(semaphore);
}
return kr;
}
kern_return_t
semaphore_timedwait(
semaphore_t semaphore,
mach_timespec_t wait_time)
{
int option = SEMAPHORE_OPTION_NONE;
uint64_t deadline = 0;
if (semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
if (BAD_MACH_TIMESPEC(&wait_time)) {
return KERN_INVALID_VALUE;
}
if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0) {
option = SEMAPHORE_TIMEOUT_NOBLOCK;
} else {
deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
}
return semaphore_wait_internal(semaphore,
SEMAPHORE_NULL,
deadline, option,
(void (*)(kern_return_t))0);
}
kern_return_t
semaphore_timedwait_trap(
struct semaphore_timedwait_trap_args *args)
{
return semaphore_timedwait_trap_internal(args->wait_name, args->sec, args->nsec, thread_syscall_return);
}
kern_return_t
semaphore_timedwait_trap_internal(
mach_port_name_t name,
unsigned int sec,
clock_res_t nsec,
void (*caller_cont)(kern_return_t))
{
semaphore_t semaphore;
mach_timespec_t wait_time;
kern_return_t kr;
wait_time.tv_sec = sec;
wait_time.tv_nsec = nsec;
if (BAD_MACH_TIMESPEC(&wait_time)) {
return KERN_INVALID_VALUE;
}
kr = port_name_to_semaphore(name, &semaphore);
if (kr == KERN_SUCCESS) {
int option = SEMAPHORE_OPTION_NONE;
uint64_t deadline = 0;
if (sec == 0 && nsec == 0) {
option = SEMAPHORE_TIMEOUT_NOBLOCK;
} else {
deadline = semaphore_deadline(sec, nsec);
}
kr = semaphore_wait_internal(semaphore,
SEMAPHORE_NULL,
deadline, option,
caller_cont);
semaphore_dereference(semaphore);
}
return kr;
}
kern_return_t
semaphore_wait_signal(
semaphore_t wait_semaphore,
semaphore_t signal_semaphore)
{
if (wait_semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
return semaphore_wait_internal(wait_semaphore,
signal_semaphore,
0ULL, SEMAPHORE_OPTION_NONE,
(void (*)(kern_return_t))0);
}
kern_return_t
semaphore_wait_signal_trap(
struct semaphore_wait_signal_trap_args *args)
{
return semaphore_wait_signal_trap_internal(args->wait_name, args->signal_name, thread_syscall_return);
}
kern_return_t
semaphore_wait_signal_trap_internal(
mach_port_name_t wait_name,
mach_port_name_t signal_name,
void (*caller_cont)(kern_return_t))
{
semaphore_t wait_semaphore;
semaphore_t signal_semaphore;
kern_return_t kr;
kr = port_name_to_semaphore(signal_name, &signal_semaphore);
if (kr == KERN_SUCCESS) {
kr = port_name_to_semaphore(wait_name, &wait_semaphore);
if (kr == KERN_SUCCESS) {
kr = semaphore_wait_internal(wait_semaphore,
signal_semaphore,
0ULL, SEMAPHORE_OPTION_NONE,
caller_cont);
semaphore_dereference(wait_semaphore);
}
semaphore_dereference(signal_semaphore);
}
return kr;
}
kern_return_t
semaphore_timedwait_signal(
semaphore_t wait_semaphore,
semaphore_t signal_semaphore,
mach_timespec_t wait_time)
{
int option = SEMAPHORE_OPTION_NONE;
uint64_t deadline = 0;
if (wait_semaphore == SEMAPHORE_NULL) {
return KERN_INVALID_ARGUMENT;
}
if (BAD_MACH_TIMESPEC(&wait_time)) {
return KERN_INVALID_VALUE;
}
if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0) {
option = SEMAPHORE_TIMEOUT_NOBLOCK;
} else {
deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
}
return semaphore_wait_internal(wait_semaphore,
signal_semaphore,
deadline, option,
(void (*)(kern_return_t))0);
}
kern_return_t
semaphore_timedwait_signal_trap(
struct semaphore_timedwait_signal_trap_args *args)
{
return semaphore_timedwait_signal_trap_internal(args->wait_name, args->signal_name, args->sec, args->nsec, thread_syscall_return);
}
kern_return_t
semaphore_timedwait_signal_trap_internal(
mach_port_name_t wait_name,
mach_port_name_t signal_name,
unsigned int sec,
clock_res_t nsec,
void (*caller_cont)(kern_return_t))
{
semaphore_t wait_semaphore;
semaphore_t signal_semaphore;
mach_timespec_t wait_time;
kern_return_t kr;
wait_time.tv_sec = sec;
wait_time.tv_nsec = nsec;
if (BAD_MACH_TIMESPEC(&wait_time)) {
return KERN_INVALID_VALUE;
}
kr = port_name_to_semaphore(signal_name, &signal_semaphore);
if (kr == KERN_SUCCESS) {
kr = port_name_to_semaphore(wait_name, &wait_semaphore);
if (kr == KERN_SUCCESS) {
int option = SEMAPHORE_OPTION_NONE;
uint64_t deadline = 0;
if (sec == 0 && nsec == 0) {
option = SEMAPHORE_TIMEOUT_NOBLOCK;
} else {
deadline = semaphore_deadline(sec, nsec);
}
kr = semaphore_wait_internal(wait_semaphore,
signal_semaphore,
deadline, option,
caller_cont);
semaphore_dereference(wait_semaphore);
}
semaphore_dereference(signal_semaphore);
}
return kr;
}
void
semaphore_reference(
semaphore_t semaphore)
{
os_ref_retain(&semaphore->ref_count);
}
void
semaphore_dereference(
semaphore_t semaphore)
{
uint32_t collisions;
spl_t spl_level;
if (semaphore == NULL) {
return;
}
if (os_ref_release(&semaphore->ref_count) > 0) {
return;
}
ipc_port_t port = semaphore->port;
if (IP_VALID(port)) {
assert(!port->ip_srights);
ipc_port_dealloc_kernel(port);
}
spl_level = splsched();
semaphore_lock(semaphore);
for (collisions = 0; semaphore->active; collisions++) {
task_t task = semaphore->owner;
assert(task != TASK_NULL);
if (task_lock_try(task)) {
semaphore_destroy_internal(task, semaphore);
splx(spl_level);
task_unlock(task);
goto out;
}
semaphore_unlock(semaphore);
splx(spl_level);
mutex_pause(collisions);
spl_level = splsched();
semaphore_lock(semaphore);
}
semaphore_unlock(semaphore);
splx(spl_level);
out:
zfree(semaphore_zone, semaphore);
}
#define WAITQ_TO_SEMA(wq) ((semaphore_t) ((uintptr_t)(wq) - offsetof(struct semaphore, waitq)))
void
kdp_sema_find_owner(struct waitq * waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo)
{
semaphore_t sem = WAITQ_TO_SEMA(waitq);
assert(event == SEMAPHORE_EVENT);
zone_require(semaphore_zone, sem);
waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(sem->port);
if (sem->owner) {
waitinfo->owner = pid_from_task(sem->owner);
}
}