/* * Copyright (c) 2003-2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _KERN_LOCKS_H_ #define _KERN_LOCKS_H_ #include <sys/cdefs.h> #include <sys/appleapiopts.h> #include <mach/boolean.h> #include <mach/mach_types.h> #include <kern/kern_types.h> #include <kern/lock_group.h> #include <machine/locks.h> __BEGIN_DECLS typedef unsigned int lck_sleep_action_t; #define LCK_SLEEP_DEFAULT 0x00 /* Release the lock while waiting for the event, then reclaim */ /* RW locks are returned in the same mode */ #define LCK_SLEEP_UNLOCK 0x01 /* Release the lock and return unheld */ #define LCK_SLEEP_SHARED 0x02 /* Reclaim the lock in shared mode (RW only) */ #define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */ #define LCK_SLEEP_SPIN 0x08 /* Reclaim the lock in spin mode (mutex only) */ #define LCK_SLEEP_PROMOTED_PRI 0x10 /* Sleep at a promoted priority */ #define LCK_SLEEP_SPIN_ALWAYS 0x20 /* Reclaim the lock in spin-always mode (mutex only) */ #define LCK_SLEEP_MASK 0x3f /* Valid actions */ typedef unsigned int lck_wake_action_t; #define LCK_WAKE_DEFAULT 0x00 /* If waiters are present, transfer their push to the wokenup thread */ #define LCK_WAKE_DO_NOT_TRANSFER_PUSH 0x01 /* Do not transfer waiters push when waking up */ #ifdef XNU_KERNEL_PRIVATE #include <kern/startup.h> typedef struct _lck_attr_ { unsigned int lck_attr_val; } lck_attr_t; extern lck_attr_t LockDefaultLckAttr; #define LCK_ATTR_NONE 0 #define LCK_ATTR_DEBUG 0x00000001 #define LCK_ATTR_RW_SHARED_PRIORITY 0x00010000 #else /* !XNU_KERNEL_PRIVATE */ typedef struct __lck_attr__ lck_attr_t; #endif /* !XNU_KERNEL_PRIVATE */ #define LCK_ATTR_NULL (lck_attr_t *)NULL extern lck_attr_t *lck_attr_alloc_init(void); extern void lck_attr_setdefault( lck_attr_t *attr); extern void lck_attr_setdebug( lck_attr_t *attr); extern void lck_attr_cleardebug( lck_attr_t *attr); #ifdef XNU_KERNEL_PRIVATE #if __x86_64__ /* * Extended mutexes are only implemented on x86_64 */ #define HAS_EXT_MUTEXES 1 #endif /* __x86_64__ */ typedef union { uint16_t tcurnext; struct { uint8_t cticket; uint8_t nticket; }; } lck_ticket_internal; typedef struct { lck_ticket_internal tu; uintptr_t lck_owner; } lck_ticket_t; void lck_ticket_init(lck_ticket_t *tlock, lck_grp_t *grp); #if LOCK_STATS void lck_ticket_lock(lck_ticket_t *tlock, lck_grp_t *grp); #else void lck_ticket_lock(lck_ticket_t *tlock); #define lck_ticket_lock(tlock, grp) lck_ticket_lock(tlock) #endif /* LOCK_STATS */ void lck_ticket_unlock(lck_ticket_t *tlock); void lck_ticket_assert_owned(lck_ticket_t *tlock); extern void lck_attr_rw_shared_priority( lck_attr_t *attr); #endif extern void lck_attr_free( lck_attr_t *attr); #define decl_lck_spin_data(class, name) class lck_spin_t name extern lck_spin_t *lck_spin_alloc_init( lck_grp_t *grp, lck_attr_t *attr); extern void lck_spin_init( lck_spin_t *lck, lck_grp_t *grp, lck_attr_t *attr); extern void lck_spin_lock( lck_spin_t *lck); extern void lck_spin_lock_grp( lck_spin_t *lck, lck_grp_t *grp); extern void lck_spin_unlock( lck_spin_t *lck); extern void lck_spin_destroy( lck_spin_t *lck, lck_grp_t *grp); extern void lck_spin_free( lck_spin_t *lck, lck_grp_t *grp); extern wait_result_t lck_spin_sleep( lck_spin_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible); extern wait_result_t lck_spin_sleep_grp( lck_spin_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, lck_grp_t *grp); extern wait_result_t lck_spin_sleep_deadline( lck_spin_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline); #ifdef KERNEL_PRIVATE extern void lck_spin_lock_nopreempt( lck_spin_t *lck); extern void lck_spin_lock_nopreempt_grp( lck_spin_t *lck, lck_grp_t *grp); extern void lck_spin_unlock_nopreempt( lck_spin_t *lck); extern boolean_t lck_spin_try_lock_grp( lck_spin_t *lck, lck_grp_t *grp); extern boolean_t lck_spin_try_lock( lck_spin_t *lck); extern boolean_t lck_spin_try_lock_nopreempt( lck_spin_t *lck); extern boolean_t lck_spin_try_lock_nopreempt_grp( lck_spin_t *lck, lck_grp_t *grp); /* NOT SAFE: To be used only by kernel debugger to avoid deadlock. */ extern boolean_t kdp_lck_spin_is_acquired( lck_spin_t *lck); struct _lck_mtx_ext_; extern void lck_mtx_init_ext( lck_mtx_t *lck, struct _lck_mtx_ext_ *lck_ext, lck_grp_t *grp, lck_attr_t *attr); #endif #define decl_lck_mtx_data(class, name) class lck_mtx_t name extern lck_mtx_t *lck_mtx_alloc_init( lck_grp_t *grp, lck_attr_t *attr); extern void lck_mtx_init( lck_mtx_t *lck, lck_grp_t *grp, lck_attr_t *attr); extern void lck_mtx_lock( lck_mtx_t *lck); extern void lck_mtx_unlock( lck_mtx_t *lck); extern void lck_mtx_destroy( lck_mtx_t *lck, lck_grp_t *grp); extern void lck_mtx_free( lck_mtx_t *lck, lck_grp_t *grp); extern wait_result_t lck_mtx_sleep( lck_mtx_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible); extern wait_result_t lck_mtx_sleep_deadline( lck_mtx_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline); #ifdef KERNEL_PRIVATE /* * Name: lck_spin_sleep_with_inheritor * * Description: deschedule the current thread and wait on the waitq associated with event to be woken up. * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will * be directed to the inheritor specified. * An interruptible mode and deadline can be specified to return earlier from the wait. * * Args: * Arg1: lck_spin_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified. * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK. * Arg3: event to wait on. * Arg4: thread to propagate the event push to. * Arg5: interruptible flag for wait. * Arg6: deadline for wait. * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The inheritor specified cannot run in user space until another inheritor is specified for the event or a * wakeup for the event is called. * * Returns: result of the wait. */ extern wait_result_t lck_spin_sleep_with_inheritor(lck_spin_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline); /* * Name: lck_mtx_sleep_with_inheritor * * Description: deschedule the current thread and wait on the waitq associated with event to be woken up. * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will * be directed to the inheritor specified. * An interruptible mode and deadline can be specified to return earlier from the wait. * * Args: * Arg1: lck_mtx_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified. * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS. * Arg3: event to wait on. * Arg4: thread to propagate the event push to. * Arg5: interruptible flag for wait. * Arg6: deadline for wait. * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The inheritor specified cannot run in user space until another inheritor is specified for the event or a * wakeup for the event is called. * * Returns: result of the wait. */ extern wait_result_t lck_mtx_sleep_with_inheritor(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline); /* * Name: lck_mtx_sleep_with_inheritor * * Description: deschedule the current thread and wait on the waitq associated with event to be woken up. * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will * be directed to the inheritor specified. * An interruptible mode and deadline can be specified to return earlier from the wait. * * Args: * Arg1: lck_rw_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified. * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE. * Arg3: event to wait on. * Arg4: thread to propagate the event push to. * Arg5: interruptible flag for wait. * Arg6: deadline for wait. * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The inheritor specified cannot run in user space until another inheritor is specified for the event or a * wakeup for the event is called. * * Returns: result of the wait. */ extern wait_result_t lck_rw_sleep_with_inheritor(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline); /* * Name: wakeup_one_with_inheritor * * Description: wake up one waiter for event if any. The thread woken up will be the one with the higher sched priority waiting on event. * The push for the event will be transferred from the last inheritor to the woken up thread. * * Args: * Arg1: event to wake from. * Arg2: wait result to pass to the woken up thread. * Arg3: pointer for storing the thread wokenup. * * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. * * Conditions: The new inheritor wokenup cannot run in user space until another inheritor is specified for the event or a * wakeup for the event is called. * A reference for the wokenup thread is acquired. * NOTE: this cannot be called from interrupt context. */ extern kern_return_t wakeup_one_with_inheritor(event_t event, wait_result_t result, lck_wake_action_t action, thread_t *thread_wokenup); /* * Name: wakeup_all_with_inheritor * * Description: wake up all waiters waiting for event. The old inheritor will lose the push. * * Args: * Arg1: event to wake from. * Arg2: wait result to pass to the woken up threads. * * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. * * Conditions: NOTE: this cannot be called from interrupt context. */ extern kern_return_t wakeup_all_with_inheritor(event_t event, wait_result_t result); /* * Name: change_sleep_inheritor * * Description: Redirect the push of the waiting threads of event to the new inheritor specified. * * Args: * Arg1: event to redirect the push. * Arg2: new inheritor for event. * * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. * * Conditions: In case of success, the new inheritor cannot run in user space until another inheritor is specified for the event or a * wakeup for the event is called. * NOTE: this cannot be called from interrupt context. */ extern kern_return_t change_sleep_inheritor(event_t event, thread_t inheritor); /* * gate structure */ typedef struct gate { uintptr_t gate_data; // thread holder, interlock bit and waiter bit struct turnstile *turnstile; // protected by the interlock bit } gate_t; #define GATE_ILOCK_BIT 0 #define GATE_WAITERS_BIT 1 #define GATE_ILOCK (1 << GATE_ILOCK_BIT) #define GATE_WAITERS (1 << GATE_WAITERS_BIT) #define gate_ilock(gate) hw_lock_bit((hw_lock_bit_t*)(&(gate)->gate_data), GATE_ILOCK_BIT, LCK_GRP_NULL) #define gate_iunlock(gate) hw_unlock_bit((hw_lock_bit_t*)(&(gate)->gate_data), GATE_ILOCK_BIT) #define gate_has_waiters(state) ((state & GATE_WAITERS) != 0) #define ordered_load_gate(gate) os_atomic_load(&(gate)->gate_data, compiler_acq_rel) #define ordered_store_gate(gate, value) os_atomic_store(&(gate)->gate_data, value, compiler_acq_rel) #define GATE_THREAD_MASK (~(uintptr_t)(GATE_ILOCK | GATE_WAITERS)) #define GATE_STATE_TO_THREAD(state) (thread_t)(state & GATE_THREAD_MASK) #define GATE_THREAD_TO_STATE(thread) ((uintptr_t)thread) /* * Possible gate_wait_result_t values. */ typedef int gate_wait_result_t; #define GATE_HANDOFF 0 #define GATE_OPENED 1 #define GATE_TIMED_OUT 2 #define GATE_INTERRUPTED 3 /* * Gate flags used by gate_assert */ #define GATE_ASSERT_CLOSED 0 #define GATE_ASSERT_OPEN 1 #define GATE_ASSERT_HELD 2 /* * Gate flags used by gate_handoff */ #define GATE_HANDOFF_DEFAULT 0 #define GATE_HANDOFF_OPEN_IF_NO_WAITERS 1 #define GATE_EVENT(gate) ((event_t) gate) #define EVENT_TO_GATE(event) ((gate_t *) event) /* * Name: decl_lck_rw_gate_data * * Description: declares a gate variable with specified storage class. * The gate itself will be stored in this variable and it is the caller's responsibility * to ensure that this variable's memory is going to be accessible by all threads that will use * the gate. * Every gate function will require a pointer to this variable as parameter. The same pointer should * be used in every thread. * * The variable needs to be initialized once with lck_rw_gate_init() and destroyed once with * lck_rw_gate_destroy() when not needed anymore. * * The gate will be used in conjunction with a lck_rw_t. * * Args: * Arg1: storage class. * Arg2: variable name. */ #define decl_lck_rw_gate_data(class, name) class gate_t name /* * Name: lck_rw_gate_init * * Description: initializes a variable declared with decl_lck_rw_gate_data. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. */ extern void lck_rw_gate_init(lck_rw_t *lock, gate_t *gate); /* * Name: lck_rw_gate_destroy * * Description: destroys a variable previously initialized. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. */ extern void lck_rw_gate_destroy(lck_rw_t *lock, gate_t *gate); /* * Name: lck_rw_gate_try_close * * Description: Tries to close the gate. * In case of success the current thread will be set as * the holder of the gate. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * * Returns: * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder * of the gate. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped * between the calls to lck_rw_gate_try_close() and lck_rw_gate_wait(). * * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate. * lck_rw_gate_wait() should be called instead if the intent is to unconditionally wait on this gate. * The calls to lck_rw_gate_try_close() and lck_rw_gate_wait() should * be done without dropping the lock that is protecting the gate in between. */ extern kern_return_t lck_rw_gate_try_close(lck_rw_t *lock, gate_t *gate); /* * Name: lck_rw_gate_close * * Description: Closes the gate. The current thread will be set as * the holder of the gate. Will panic if the gate is already closed. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be open. * */ extern void lck_rw_gate_close(lck_rw_t *lock, gate_t *gate); /* * Name: lck_rw_gate_open * * Description: Opens the gate and wakes up possible waiters. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * */ extern void lck_rw_gate_open(lck_rw_t *lock, gate_t *gate); /* * Name: lck_rw_gate_handoff * * Description: Tries to transfer the ownership of the gate. The waiter with highest sched * priority will be selected as the new holder of the gate, and woken up, * with the gate remaining in the closed state throughout. * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING * will be returned. * GATE_HANDOFF_OPEN_IF_NO_WAITERS flag can be used to specify if the gate should be opened in * case no waiters were found. * * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * Arg3: flags - GATE_HANDOFF_DEFAULT or GATE_HANDOFF_OPEN_IF_NO_WAITERS * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * * Returns: * KERN_SUCCESS in case one of the waiters became the new holder. * KERN_NOT_WAITING in case there were no waiters. * */ extern kern_return_t lck_rw_gate_handoff(lck_rw_t *lock, gate_t *gate, int flags); /* * Name: lck_rw_gate_steal * * Description: Set the current ownership of the gate. It sets the current thread as the * new holder of the gate. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * NOTE: the previous holder should not call lck_rw_gate_open() or lck_rw_gate_handoff() * anymore. * * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be closed and the current thread must not already be the holder. * */ extern void lck_rw_gate_steal(lck_rw_t *lock, gate_t *gate); /* * Name: lck_rw_gate_wait * * Description: Waits for the current thread to become the holder of the gate or for the * gate to become open. An interruptible mode and deadline can be specified * to return earlier from the wait. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE. * Arg3: interruptible flag for wait. * Arg4: deadline * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The gate must be closed. * * Returns: Reason why the thread was woken up. * GATE_HANDOFF - the current thread was handed off the ownership of the gate. * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * GATE_OPENED - the gate was opened by the holder. * GATE_TIMED_OUT - the thread was woken up by a timeout. * GATE_INTERRUPTED - the thread was interrupted while sleeping. * */ extern gate_wait_result_t lck_rw_gate_wait(lck_rw_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline); /* * Name: lck_rw_gate_assert * * Description: asserts that the gate is in the specified state. * * Args: * Arg1: lck_rw_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. * Arg3: flags to specified assert type. * GATE_ASSERT_CLOSED - the gate is currently closed * GATE_ASSERT_OPEN - the gate is currently opened * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder */ extern void lck_rw_gate_assert(lck_rw_t *lock, gate_t *gate, int flags); /* * Name: decl_lck_mtx_gate_data * * Description: declares a gate variable with specified storage class. * The gate itself will be stored in this variable and it is the caller's responsibility * to ensure that this variable's memory is going to be accessible by all threads that will use * the gate. * Every gate function will require a pointer to this variable as parameter. The same pointer should * be used in every thread. * * The variable needs to be initialized once with lck_mtx_gate_init() and destroyed once with * lck_mtx_gate_destroy() when not needed anymore. * * The gate will be used in conjunction with a lck_mtx_t. * * Args: * Arg1: storage class. * Arg2: variable name. */ #define decl_lck_mtx_gate_data(class, name) class gate_t name /* * Name: lck_mtx_gate_init * * Description: initializes a variable declared with decl_lck_mtx_gate_data. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. */ extern void lck_mtx_gate_init(lck_mtx_t *lock, gate_t *gate); /* * Name: lck_mtx_gate_destroy * * Description: destroys a variable previously initialized * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. */ extern void lck_mtx_gate_destroy(lck_mtx_t *lock, gate_t *gate); /* * Name: lck_mtx_gate_try_close * * Description: Tries to close the gate. * In case of success the current thread will be set as * the holder of the gate. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * * Returns: * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder * of the gate. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped * between the calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait(). * * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate. * lck_mtx_gate_wait() should be called instead if the intent is to unconditionally wait on this gate. * The calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait() should * be done without dropping the lock that is protecting the gate in between. */ extern kern_return_t lck_mtx_gate_try_close(lck_mtx_t *lock, gate_t *gate); /* * Name: lck_mtx_gate_close * * Description: Closes the gate. The current thread will be set as * the holder of the gate. Will panic if the gate is already closed. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be open. * */ extern void lck_mtx_gate_close(lck_mtx_t *lock, gate_t *gate); /* * Name: lck_mtx_gate_open * * Description: Opens of the gate and wakes up possible waiters. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * */ extern void lck_mtx_gate_open(lck_mtx_t *lock, gate_t *gate); /* * Name: lck_mtx_gate_handoff * * Description: Set the current ownership of the gate. The waiter with highest sched * priority will be selected as the new holder of the gate, and woken up, * with the gate remaining in the closed state throughout. * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING * will be returned. * OPEN_ON_FAILURE flag can be used to specify if the gate should be opened in * case no waiters were found. * * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * Arg3: flags - GATE_NO_FALGS or OPEN_ON_FAILURE * * Conditions: Lock must be held. Returns with the lock held. * The current thread must be the holder of the gate. * * Returns: * KERN_SUCCESS in case one of the waiters became the new holder. * KERN_NOT_WAITING in case there were no waiters. * */ extern kern_return_t lck_mtx_gate_handoff(lck_mtx_t *lock, gate_t *gate, int flags); /* * Name: lck_mtx_gate_steal * * Description: Steals the ownership of the gate. It sets the current thread as the * new holder of the gate. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * NOTE: the previous holder should not call lck_mtx_gate_open() or lck_mtx_gate_handoff() * anymore. * * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * * Conditions: Lock must be held. Returns with the lock held. * The gate must be closed and the current thread must not already be the holder. * */ extern void lck_mtx_gate_steal(lck_mtx_t *lock, gate_t *gate); /* * Name: lck_mtx_gate_wait * * Description: Waits for the current thread to become the holder of the gate or for the * gate to become open. An interruptible mode and deadline can be specified * to return earlier from the wait. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS. * Arg3: interruptible flag for wait. * Arg4: deadline * * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. * Lock will be dropped while waiting. * The gate must be closed. * * Returns: Reason why the thread was woken up. * GATE_HANDOFF - the current thread was handed off the ownership of the gate. * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on * to wake up possible waiters on the gate before returning to userspace. * GATE_OPENED - the gate was opened by the holder. * GATE_TIMED_OUT - the thread was woken up by a timeout. * GATE_INTERRUPTED - the thread was interrupted while sleeping. * */ extern gate_wait_result_t lck_mtx_gate_wait(lck_mtx_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline); /* * Name: lck_mtx_gate_assert * * Description: asserts that the gate is in the specified state. * * Args: * Arg1: lck_mtx_t lock used to protect the gate. * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. * Arg3: flags to specified assert type. * GATE_ASSERT_CLOSED - the gate is currently closed * GATE_ASSERT_OPEN - the gate is currently opened * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder */ extern void lck_mtx_gate_assert(lck_mtx_t *lock, gate_t *gate, int flags); #endif //KERNEL_PRIVATE #if DEVELOPMENT || DEBUG #define FULL_CONTENDED 0 #define HALF_CONTENDED 1 #define MAX_CONDENDED 2 extern void erase_all_test_mtx_stats(void); extern int get_test_mtx_stats_string(char* buffer, int buffer_size); extern void lck_mtx_test_init(void); extern void lck_mtx_test_lock(void); extern void lck_mtx_test_unlock(void); extern int lck_mtx_test_mtx_uncontended(int iter, char* buffer, int buffer_size); extern int lck_mtx_test_mtx_contended(int iter, char* buffer, int buffer_size, int type); extern int lck_mtx_test_mtx_uncontended_loop_time(int iter, char* buffer, int buffer_size); extern int lck_mtx_test_mtx_contended_loop_time(int iter, char* buffer, int buffer_size, int type); #endif #ifdef KERNEL_PRIVATE extern boolean_t lck_mtx_try_lock( lck_mtx_t *lck); extern void mutex_pause(uint32_t); extern void lck_mtx_yield( lck_mtx_t *lck); extern boolean_t lck_mtx_try_lock_spin( lck_mtx_t *lck); extern void lck_mtx_lock_spin( lck_mtx_t *lck); extern boolean_t kdp_lck_mtx_lock_spin_is_acquired( lck_mtx_t *lck); extern void lck_mtx_convert_spin( lck_mtx_t *lck); extern void lck_mtx_lock_spin_always( lck_mtx_t *lck); extern boolean_t lck_mtx_try_lock_spin_always( lck_mtx_t *lck); #define lck_mtx_unlock_always(l) lck_mtx_unlock(l) extern void lck_spin_assert( lck_spin_t *lck, unsigned int type); extern boolean_t kdp_lck_rw_lock_is_acquired_exclusive( lck_rw_t *lck); #endif /* KERNEL_PRIVATE */ extern void lck_mtx_assert( lck_mtx_t *lck, unsigned int type); #if MACH_ASSERT #define LCK_MTX_ASSERT(lck, type) lck_mtx_assert((lck),(type)) #define LCK_SPIN_ASSERT(lck, type) lck_spin_assert((lck),(type)) #define LCK_RW_ASSERT(lck, type) lck_rw_assert((lck),(type)) #else /* MACH_ASSERT */ #define LCK_MTX_ASSERT(lck, type) #define LCK_SPIN_ASSERT(lck, type) #define LCK_RW_ASSERT(lck, type) #endif /* MACH_ASSERT */ #if DEBUG #define LCK_MTX_ASSERT_DEBUG(lck, type) lck_mtx_assert((lck),(type)) #define LCK_SPIN_ASSERT_DEBUG(lck, type) lck_spin_assert((lck),(type)) #define LCK_RW_ASSERT_DEBUG(lck, type) lck_rw_assert((lck),(type)) #else /* DEBUG */ #define LCK_MTX_ASSERT_DEBUG(lck, type) #define LCK_SPIN_ASSERT_DEBUG(lck, type) #define LCK_RW_ASSERT_DEBUG(lck, type) #endif /* DEBUG */ #define LCK_ASSERT_OWNED 1 #define LCK_ASSERT_NOTOWNED 2 #define LCK_MTX_ASSERT_OWNED LCK_ASSERT_OWNED #define LCK_MTX_ASSERT_NOTOWNED LCK_ASSERT_NOTOWNED #ifdef MACH_KERNEL_PRIVATE struct turnstile; extern void lck_mtx_lock_wait( lck_mtx_t *lck, thread_t holder, struct turnstile **ts); extern int lck_mtx_lock_acquire( lck_mtx_t *lck, struct turnstile *ts); extern boolean_t lck_mtx_unlock_wakeup( lck_mtx_t *lck, thread_t holder); extern boolean_t lck_mtx_ilk_unlock( lck_mtx_t *lck); extern boolean_t lck_mtx_ilk_try_lock( lck_mtx_t *lck); extern void lck_mtx_wakeup_adjust_pri(thread_t thread, integer_t priority); #endif #define decl_lck_rw_data(class, name) class lck_rw_t name typedef unsigned int lck_rw_type_t; #define LCK_RW_TYPE_SHARED 0x01 #define LCK_RW_TYPE_EXCLUSIVE 0x02 #ifdef XNU_KERNEL_PRIVATE #define LCK_RW_ASSERT_SHARED 0x01 #define LCK_RW_ASSERT_EXCLUSIVE 0x02 #define LCK_RW_ASSERT_HELD 0x03 #define LCK_RW_ASSERT_NOTHELD 0x04 #endif extern lck_rw_t *lck_rw_alloc_init( lck_grp_t *grp, lck_attr_t *attr); extern void lck_rw_init( lck_rw_t *lck, lck_grp_t *grp, lck_attr_t *attr); extern void lck_rw_lock( lck_rw_t *lck, lck_rw_type_t lck_rw_type); extern void lck_rw_unlock( lck_rw_t *lck, lck_rw_type_t lck_rw_type); extern void lck_rw_lock_shared( lck_rw_t *lck); extern void lck_rw_unlock_shared( lck_rw_t *lck); extern boolean_t lck_rw_lock_yield_shared( lck_rw_t *lck, boolean_t force_yield); extern void lck_rw_lock_exclusive( lck_rw_t *lck); /* * Grabs the lock exclusive. * Returns true iff the thread spun or blocked while attempting to * acquire the lock. * * Note that the return value is ONLY A HEURISTIC w.r.t. the lock's * contention. * * This routine IS EXPERIMENTAL. * It's only used for the vm object lock, and use for other subsystems * is UNSUPPORTED. */ extern bool lck_rw_lock_exclusive_check_contended( lck_rw_t *lck); extern void lck_rw_unlock_exclusive( lck_rw_t *lck); #ifdef XNU_KERNEL_PRIVATE /* * CAUTION * read-write locks do not have a concept of ownership, so lck_rw_assert() * merely asserts that someone is holding the lock, not necessarily the caller. */ extern void lck_rw_assert( lck_rw_t *lck, unsigned int type); extern void lck_rw_clear_promotion(thread_t thread, uintptr_t trace_obj); extern void lck_rw_set_promotion_locked(thread_t thread); uintptr_t unslide_for_kdebug(void* object); #endif /* XNU_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE extern lck_rw_type_t lck_rw_done( lck_rw_t *lck); #endif extern void lck_rw_destroy( lck_rw_t *lck, lck_grp_t *grp); extern void lck_rw_free( lck_rw_t *lck, lck_grp_t *grp); extern wait_result_t lck_rw_sleep( lck_rw_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible); extern wait_result_t lck_rw_sleep_deadline( lck_rw_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline); extern boolean_t lck_rw_lock_shared_to_exclusive( lck_rw_t *lck); extern void lck_rw_lock_exclusive_to_shared( lck_rw_t *lck); extern boolean_t lck_rw_try_lock( lck_rw_t *lck, lck_rw_type_t lck_rw_type); #ifdef KERNEL_PRIVATE extern boolean_t lck_rw_try_lock_shared( lck_rw_t *lck); extern boolean_t lck_rw_try_lock_exclusive( lck_rw_t *lck); #endif #if XNU_KERNEL_PRIVATE struct lck_attr_startup_spec { lck_attr_t *lck_attr; uint32_t lck_attr_set_flags; uint32_t lck_attr_clear_flags; }; struct lck_spin_startup_spec { lck_spin_t *lck; lck_grp_t *lck_grp; lck_attr_t *lck_attr; }; struct lck_mtx_startup_spec { lck_mtx_t *lck; struct _lck_mtx_ext_ *lck_ext; lck_grp_t *lck_grp; lck_attr_t *lck_attr; }; struct lck_rw_startup_spec { lck_rw_t *lck; lck_grp_t *lck_grp; lck_attr_t *lck_attr; }; extern void lck_attr_startup_init( struct lck_attr_startup_spec *spec); extern void lck_spin_startup_init( struct lck_spin_startup_spec *spec); extern void lck_mtx_startup_init( struct lck_mtx_startup_spec *spec); extern void lck_rw_startup_init( struct lck_rw_startup_spec *spec); /* * Auto-initializing locks declarations * ------------------------------------ * * Unless you need to configure your locks in very specific ways, * there is no point creating explicit lock attributes. For most * static locks, these declaration macros can be used: * * - LCK_SPIN_DECLARE for spinlocks, * - LCK_MTX_EARLY_DECLARE for mutexes initialized before memory * allocations are possible, * - LCK_MTX_DECLARE for mutexes, * - LCK_RW_DECLARE for reader writer locks. * * For cases when some particular attributes need to be used, * these come in *_ATTR variants that take a variable declared with * LCK_ATTR_DECLARE as an argument. */ #define LCK_ATTR_DECLARE(var, set_flags, clear_flags) \ SECURITY_READ_ONLY_LATE(lck_attr_t) var; \ static __startup_data struct lck_attr_startup_spec \ __startup_lck_attr_spec_ ## var = { &var, set_flags, clear_flags }; \ STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_SECOND, lck_attr_startup_init, \ &__startup_lck_attr_spec_ ## var) #define LCK_SPIN_DECLARE_ATTR(var, grp, attr) \ lck_spin_t var; \ static __startup_data struct lck_spin_startup_spec \ __startup_lck_spin_spec_ ## var = { &var, grp, attr }; \ STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_FOURTH, lck_spin_startup_init, \ &__startup_lck_spin_spec_ ## var) #define LCK_SPIN_DECLARE(var, grp) \ LCK_SPIN_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) #define LCK_MTX_DECLARE_ATTR(var, grp, attr) \ lck_mtx_t var; \ static __startup_data struct lck_mtx_startup_spec \ __startup_lck_mtx_spec_ ## var = { &var, NULL, grp, attr }; \ STARTUP_ARG(LOCKS, STARTUP_RANK_FIRST, lck_mtx_startup_init, \ &__startup_lck_mtx_spec_ ## var) #define LCK_MTX_DECLARE(var, grp) \ LCK_MTX_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) #define LCK_MTX_EARLY_DECLARE_ATTR(var, grp, attr) \ lck_mtx_ext_t var ## _ext; \ lck_mtx_t var; \ static __startup_data struct lck_mtx_startup_spec \ __startup_lck_mtx_spec_ ## var = { &var, &var ## _ext, grp, attr }; \ STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_FOURTH, lck_mtx_startup_init, \ &__startup_lck_mtx_spec_ ## var) #define LCK_MTX_EARLY_DECLARE(var, grp) \ LCK_MTX_EARLY_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) #define LCK_RW_DECLARE_ATTR(var, grp, attr) \ lck_rw_t var; \ static __startup_data struct lck_rw_startup_spec \ __startup_lck_rw_spec_ ## var = { &var, grp, attr }; \ STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_FOURTH, lck_rw_startup_init, \ &__startup_lck_rw_spec_ ## var) #define LCK_RW_DECLARE(var, grp) \ LCK_RW_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) #endif /* XNU_KERNEL_PRIVATE */ __END_DECLS #endif /* _KERN_LOCKS_H_ */