#include "resolver.h"
#include "internal.h"
#include "kern/kern_trace.h"
#include <sys/syscall.h>
#include "os/atomic.h"
#ifdef PLOCKSTAT
#include "plockstat.h"
#else
#define PLOCKSTAT_MUTEX_SPIN(x)
#define PLOCKSTAT_MUTEX_SPUN(x, y, z)
#define PLOCKSTAT_MUTEX_ERROR(x, y)
#define PLOCKSTAT_MUTEX_BLOCK(x)
#define PLOCKSTAT_MUTEX_BLOCKED(x, y)
#define PLOCKSTAT_MUTEX_ACQUIRE(x, y, z)
#define PLOCKSTAT_MUTEX_RELEASE(x, y)
#endif
#define PTHREAD_MUTEX_INIT_UNUSED 1
extern int __unix_conforming;
#ifndef BUILDING_VARIANT
PTHREAD_NOEXPORT PTHREAD_WEAK int
_pthread_mutex_unlock_slow(pthread_mutex_t *omutex);
PTHREAD_NOEXPORT PTHREAD_WEAK int
_pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock);
PTHREAD_NOEXPORT PTHREAD_WEAK int
_pthread_mutex_lock_wait(pthread_mutex_t *omutex, uint64_t newval64, uint64_t oldtid);
#endif
#define DEBUG_TRACE_POINTS 0
#if DEBUG_TRACE_POINTS
extern int __syscall(int number, ...);
#define DEBUG_TRACE(x, a, b, c, d) __syscall(SYS_kdebug_trace, TRACE_##x, a, b, c, d)
#else
#define DEBUG_TRACE(x, a, b, c, d) do { } while(0)
#endif
#include <machine/cpu_capabilities.h>
static inline int _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, uint32_t static_type);
#if !__LITTLE_ENDIAN__
#error MUTEX_GETSEQ_ADDR assumes little endian layout of 2 32-bit sequence words
#endif
PTHREAD_ALWAYS_INLINE
static inline void
MUTEX_GETSEQ_ADDR(_pthread_mutex *mutex,
volatile uint64_t **seqaddr)
{
*seqaddr = (volatile uint64_t*)(((uintptr_t)mutex->m_seq + 0x7ul) & ~0x7ul);
}
PTHREAD_ALWAYS_INLINE
static inline void
MUTEX_GETTID_ADDR(_pthread_mutex *mutex,
volatile uint64_t **tidaddr)
{
*tidaddr = (volatile uint64_t*)(((uintptr_t)mutex->m_tid + 0x7ul) & ~0x7ul);
}
#ifndef BUILDING_VARIANT
#ifndef OS_UP_VARIANT_ONLY
#define BLOCK_FAIL_PLOCKSTAT 0
#define BLOCK_SUCCESS_PLOCKSTAT 1
#ifdef PLOCKSTAT
PTHREAD_NOEXPORT PTHREAD_USED
void
_plockstat_never_fired(void)
{
PLOCKSTAT_MUTEX_SPIN(NULL);
PLOCKSTAT_MUTEX_SPUN(NULL, 0, 0);
}
#endif // PLOCKSTAT
int
pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr)
{
#if 0
if (_pthread_mutex_check_signature(mutex))
return EBUSY;
#endif
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
LOCK_INIT(mutex->lock);
return (_pthread_mutex_init(mutex, attr, 0x7));
}
int
pthread_mutex_getprioceiling(const pthread_mutex_t *omutex, int *prioceiling)
{
int res = EINVAL;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
if (_pthread_mutex_check_signature(mutex)) {
LOCK(mutex->lock);
*prioceiling = mutex->prioceiling;
res = 0;
UNLOCK(mutex->lock);
}
return res;
}
int
pthread_mutex_setprioceiling(pthread_mutex_t *omutex, int prioceiling, int *old_prioceiling)
{
int res = EINVAL;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
if (_pthread_mutex_check_signature(mutex)) {
LOCK(mutex->lock);
if (prioceiling >= -999 || prioceiling <= 999) {
*old_prioceiling = mutex->prioceiling;
mutex->prioceiling = prioceiling;
res = 0;
}
UNLOCK(mutex->lock);
}
return res;
}
int
pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *prioceiling)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
*prioceiling = attr->prioceiling;
res = 0;
}
return res;
}
int
pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int *protocol)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
*protocol = attr->protocol;
res = 0;
}
return res;
}
int
pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
*type = attr->type;
res = 0;
}
return res;
}
int
pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
*pshared = (int)attr->pshared;
res = 0;
}
return res;
}
int
pthread_mutexattr_init(pthread_mutexattr_t *attr)
{
attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;
attr->policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
attr->type = PTHREAD_MUTEX_DEFAULT;
attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
attr->pshared = _PTHREAD_DEFAULT_PSHARED;
return 0;
}
int
pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int prioceiling)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
if (prioceiling >= -999 || prioceiling <= 999) {
attr->prioceiling = prioceiling;
res = 0;
}
}
return res;
}
int
pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
switch (protocol) {
case PTHREAD_PRIO_NONE:
case PTHREAD_PRIO_INHERIT:
case PTHREAD_PRIO_PROTECT:
attr->protocol = protocol;
res = 0;
break;
}
}
return res;
}
int
pthread_mutexattr_setpolicy_np(pthread_mutexattr_t *attr, int policy)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
switch (policy) {
case _PTHREAD_MUTEX_POLICY_FAIRSHARE:
case _PTHREAD_MUTEX_POLICY_FIRSTFIT:
attr->policy = policy;
res = 0;
break;
}
}
return res;
}
int
pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
{
int res = EINVAL;
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
switch (type) {
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_RECURSIVE:
attr->type = type;
res = 0;
break;
}
}
return res;
}
void
cthread_yield(void)
{
sched_yield();
}
void
pthread_yield_np(void)
{
sched_yield();
}
int
pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
{
int res = EINVAL;
#if __DARWIN_UNIX03
if (__unix_conforming == 0) {
__unix_conforming = 1;
}
#endif
if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
#if __DARWIN_UNIX03
if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED))
#else
if ( pshared == PTHREAD_PROCESS_PRIVATE)
#endif
{
attr->pshared = pshared;
res = 0;
}
}
return res;
}
PTHREAD_NOEXPORT PTHREAD_WEAK int
_pthread_mutex_corruption_abort(_pthread_mutex *mutex);
PTHREAD_NOINLINE
int
_pthread_mutex_corruption_abort(_pthread_mutex *mutex)
{
PTHREAD_ABORT("pthread_mutex corruption: mutex %p owner changed in the middle of lock/unlock");
return EINVAL; }
PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_unlock_updatebits(_pthread_mutex *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
{
bool firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
uint32_t lgenval, ugenval, flags;
uint64_t oldtid, newtid;
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
flags = mutex->mtxopts.value;
flags &= ~_PTHREAD_MTX_OPT_NOTIFY;
if (mutex->mtxopts.options.type != PTHREAD_MUTEX_NORMAL) {
uint64_t selfid = _pthread_selfid_direct();
if (*tidaddr != selfid) {
PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, EPERM);
return EPERM;
} else if (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE &&
--mutex->mtxopts.options.lock_count) {
PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, 1);
if (flagsp != NULL) {
*flagsp = flags;
}
return 0;
}
}
uint64_t oldval64, newval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
bool clearprepost, clearnotify, spurious;
do {
oldval64 = *seqaddr;
oldtid = *tidaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
clearprepost = false;
clearnotify = false;
spurious = false;
int numwaiters = diff_genseq(lgenval, ugenval);
if (numwaiters == 0) {
spurious = true;
} else {
ugenval += PTHRW_INC;
if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
clearnotify = true;
newtid = 0; } else {
if (firstfit) {
lgenval &= ~PTH_RWL_EBIT; newtid = 0;
} else {
newtid = PTHREAD_MTX_TID_SWITCHING;
}
flags |= _PTHREAD_MTX_OPT_NOTIFY;
}
if (newtid != oldtid) {
if (!os_atomic_cmpxchg(tidaddr, oldtid, newtid, relaxed)) {
return _pthread_mutex_corruption_abort(mutex);
}
}
}
if (clearnotify || spurious) {
flags &= ~_PTHREAD_MTX_OPT_NOTIFY;
if (firstfit && ((lgenval & PTH_RWL_PBIT) != 0)) {
clearprepost = true;
lgenval &= ~PTH_RWL_PBIT;
}
}
newval64 = (((uint64_t)ugenval) << 32);
newval64 |= lgenval;
} while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
if (clearprepost) {
__psynch_cvclrprepost(mutex, lgenval, ugenval, 0, 0, lgenval, (flags | _PTHREAD_MTX_OPT_MUTEX));
}
if (mgenp != NULL) {
*mgenp = lgenval;
}
if (ugenp != NULL) {
*ugenp = ugenval;
}
if (pmtxp != NULL) {
*pmtxp = (uint32_t *)mutex;
}
if (flagsp != NULL) {
*flagsp = flags;
}
return 0;
}
PTHREAD_NOEXPORT
int
__mtx_droplock(_pthread_mutex *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp)
{
return _pthread_mutex_unlock_updatebits(mutex, flagsp, pmtxp, mgenp, ugenp);
}
PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)
{
int res = 0;
int firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
int isebit = 0;
uint32_t lgenval, ugenval;
uint64_t oldval64, newval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
uint64_t oldtid;
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
do {
do {
oldval64 = *seqaddr;
oldtid = *tidaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
} while (isebit && (lgenval & PTH_RWL_EBIT) == 0);
if (isebit) {
res = 1;
break;
}
if (firstfit) {
isebit = (lgenval & PTH_RWL_EBIT) != 0;
} else if ((lgenval & (PTH_RWL_KBIT|PTH_RWL_EBIT)) == (PTH_RWL_KBIT|PTH_RWL_EBIT)) {
break;
}
lgenval |= PTH_RWL_KBIT | PTH_RWL_EBIT;
newval64 = (((uint64_t)ugenval) << 32);
newval64 |= lgenval;
} while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire) || (firstfit && isebit));
if (res == 0) {
if (!os_atomic_cmpxchg(tidaddr, oldtid, selfid, relaxed)) {
return _pthread_mutex_corruption_abort(mutex);
}
}
return res;
}
PTHREAD_NOINLINE
static int
__mtx_markprepost(_pthread_mutex *mutex, uint32_t updateval, int firstfit)
{
uint32_t flags;
uint32_t lgenval, ugenval;
uint64_t oldval64, newval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
if (firstfit != 0 && (updateval & PTH_RWL_PBIT) != 0) {
int clearprepost;
do {
clearprepost = 0;
flags = mutex->mtxopts.value;
oldval64 = *seqaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
clearprepost = 1;
lgenval &= ~PTH_RWL_PBIT;
} else {
lgenval |= PTH_RWL_PBIT;
}
newval64 = (((uint64_t)ugenval) << 32);
newval64 |= lgenval;
} while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
if (clearprepost != 0) {
__psynch_cvclrprepost(mutex, lgenval, ugenval, 0, 0, lgenval, (flags | _PTHREAD_MTX_OPT_MUTEX));
}
}
return 0;
}
PTHREAD_NOINLINE
static int
_pthread_mutex_check_init_slow(pthread_mutex_t *omutex)
{
int res = EINVAL;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
if (_pthread_mutex_check_signature_init(mutex)) {
LOCK(mutex->lock);
if (_pthread_mutex_check_signature_init(mutex)) {
res = _pthread_mutex_init(mutex, NULL, (mutex->sig & 0xf));
} else if (_pthread_mutex_check_signature(mutex)) {
res = 0;
}
UNLOCK(mutex->lock);
} else if (_pthread_mutex_check_signature(mutex)) {
res = 0;
}
if (res != 0) {
PLOCKSTAT_MUTEX_ERROR(omutex, res);
}
return res;
}
PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_check_init(pthread_mutex_t *omutex)
{
int res = 0;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
if (!_pthread_mutex_check_signature(mutex)) {
return _pthread_mutex_check_init_slow(omutex);
}
return res;
}
PTHREAD_NOINLINE
int
_pthread_mutex_lock_wait(pthread_mutex_t *omutex, uint64_t newval64, uint64_t oldtid)
{
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
uint32_t lgenval = (uint32_t)newval64;
uint32_t ugenval = (uint32_t)(newval64 >> 32);
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
uint64_t selfid = _pthread_selfid_direct();
PLOCKSTAT_MUTEX_BLOCK(omutex);
do {
uint32_t updateval;
do {
updateval = __psynch_mutexwait(omutex, lgenval, ugenval, oldtid, mutex->mtxopts.value);
oldtid = *tidaddr;
} while (updateval == (uint32_t)-1);
} while (_pthread_mutex_lock_updatebits(mutex, selfid) == 1);
PLOCKSTAT_MUTEX_BLOCKED(omutex, BLOCK_SUCCESS_PLOCKSTAT);
return 0;
}
int
_pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock)
{
int res;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
res = _pthread_mutex_check_init(omutex);
if (res != 0) {
return res;
}
uint64_t oldtid;
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
uint64_t selfid = _pthread_selfid_direct();
if (mutex->mtxopts.options.type != PTHREAD_MUTEX_NORMAL) {
if (*tidaddr == selfid) {
if (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
if (mutex->mtxopts.options.lock_count < USHRT_MAX) {
mutex->mtxopts.options.lock_count++;
PLOCKSTAT_MUTEX_ACQUIRE(omutex, 1, 0);
res = 0;
} else {
res = EAGAIN;
PLOCKSTAT_MUTEX_ERROR(omutex, res);
}
} else if (trylock) {
res = EBUSY;
PLOCKSTAT_MUTEX_ERROR(omutex, res);
} else {
res = EDEADLK;
PLOCKSTAT_MUTEX_ERROR(omutex, res);
}
return res;
}
}
uint64_t oldval64, newval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
uint32_t lgenval, ugenval;
bool gotlock = false;
do {
oldval64 = *seqaddr;
oldtid = *tidaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
gotlock = ((lgenval & PTH_RWL_EBIT) == 0);
if (trylock && !gotlock) {
} else {
lgenval += PTHRW_INC;
lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
}
newval64 = (((uint64_t)ugenval) << 32);
newval64 |= lgenval;
} while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire));
if (gotlock) {
os_atomic_store(tidaddr, selfid, relaxed);
res = 0;
DEBUG_TRACE(psynch_mutex_ulock, omutex, lgenval, ugenval, selfid);
PLOCKSTAT_MUTEX_ACQUIRE(omutex, 0, 0);
} else if (trylock) {
res = EBUSY;
DEBUG_TRACE(psynch_mutex_utrylock_failed, omutex, lgenval, ugenval, oldtid);
PLOCKSTAT_MUTEX_ERROR(omutex, res);
} else {
res = _pthread_mutex_lock_wait(omutex, newval64, oldtid);
}
if (res == 0 && mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
mutex->mtxopts.options.lock_count = 1;
}
PLOCKSTAT_MUTEX_ACQUIRE(omutex, 0, 0);
return res;
}
#endif // OS_UP_VARIANT_ONLY
PTHREAD_ALWAYS_INLINE
static inline int
_pthread_mutex_lock(pthread_mutex_t *omutex, bool trylock)
{
#if PLOCKSTAT || DEBUG_TRACE_POINTS
if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
DEBUG_TRACE_POINTS) {
return _pthread_mutex_lock_slow(omutex, trylock);
}
#endif
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
if (!_pthread_mutex_check_signature_fast(mutex)) {
return _pthread_mutex_lock_slow(omutex, trylock);
}
uint64_t oldtid;
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
uint64_t selfid = _pthread_selfid_direct();
uint64_t oldval64, newval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
uint32_t lgenval, ugenval;
bool gotlock = false;
do {
oldval64 = *seqaddr;
oldtid = *tidaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
gotlock = ((lgenval & PTH_RWL_EBIT) == 0);
if (trylock && !gotlock) {
} else {
lgenval += PTHRW_INC;
lgenval |= PTH_RWL_EBIT | PTH_RWL_KBIT;
}
newval64 = (((uint64_t)ugenval) << 32);
newval64 |= lgenval;
} while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, acquire));
if (os_fastpath(gotlock)) {
os_atomic_store(tidaddr, selfid, relaxed);
return 0;
} else if (trylock) {
return EBUSY;
} else {
return _pthread_mutex_lock_wait(omutex, newval64, oldtid);
}
}
PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_lock(pthread_mutex_t *mutex)
{
return _pthread_mutex_lock(mutex, false);
}
PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_trylock(pthread_mutex_t *mutex)
{
return _pthread_mutex_lock(mutex, true);
}
#ifndef OS_UP_VARIANT_ONLY
PTHREAD_NOINLINE
static int
_pthread_mutex_unlock_drop(pthread_mutex_t *omutex, uint64_t newval64, uint32_t flags)
{
int res;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
uint32_t lgenval = (uint32_t)newval64;
uint32_t ugenval = (uint32_t)(newval64 >> 32);
uint32_t updateval;
int firstfit = (mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FIRSTFIT);
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
updateval = __psynch_mutexdrop(omutex, lgenval, ugenval, *tidaddr, flags);
if (updateval == (uint32_t)-1) {
res = errno;
if (res == EINTR) {
res = 0;
}
if (res != 0) {
PTHREAD_ABORT("__p_mutexdrop failed with error %d", res);
}
return res;
} else if (firstfit == 1) {
if ((updateval & PTH_RWL_PBIT) != 0) {
__mtx_markprepost(mutex, updateval, firstfit);
}
}
return 0;
}
int
_pthread_mutex_unlock_slow(pthread_mutex_t *omutex)
{
int res;
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
uint32_t mtxgen, mtxugen, flags;
res = _pthread_mutex_check_init(omutex);
if (res != 0) {
return res;
}
res = _pthread_mutex_unlock_updatebits(mutex, &flags, NULL, &mtxgen, &mtxugen);
if (res != 0) {
return res;
}
if ((flags & _PTHREAD_MTX_OPT_NOTIFY) != 0) {
uint64_t newval64;
newval64 = (((uint64_t)mtxugen) << 32);
newval64 |= mtxgen;
return _pthread_mutex_unlock_drop(omutex, newval64, flags);
} else {
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
DEBUG_TRACE(psynch_mutex_uunlock, omutex, mtxgen, mtxugen, *tidaddr);
}
return 0;
}
#endif // OS_UP_VARIANT_ONLY
PTHREAD_NOEXPORT_VARIANT
int
pthread_mutex_unlock(pthread_mutex_t *omutex)
{
#if PLOCKSTAT || DEBUG_TRACE_POINTS
if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
DEBUG_TRACE_POINTS) {
return _pthread_mutex_unlock_slow(omutex);
}
#endif
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
if (!_pthread_mutex_check_signature_fast(mutex)) {
return _pthread_mutex_unlock_slow(omutex);
}
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
uint64_t oldval64, newval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
uint32_t lgenval, ugenval;
do {
oldval64 = *seqaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
int numwaiters = diff_genseq(lgenval, ugenval);
if (numwaiters == 0) {
} else {
ugenval += PTHRW_INC;
if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
lgenval &= ~(PTH_RWL_KBIT | PTH_RWL_EBIT);
} else {
return _pthread_mutex_unlock_slow(omutex);
}
os_atomic_store(tidaddr, 0, relaxed);
}
newval64 = (((uint64_t)ugenval) << 32);
newval64 |= lgenval;
} while (!os_atomic_cmpxchg(seqaddr, oldval64, newval64, release));
return 0;
}
#ifndef OS_UP_VARIANT_ONLY
static inline int
_pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
uint32_t static_type)
{
mutex->mtxopts.value = 0;
mutex->mtxopts.options.mutex = 1;
if (attr) {
if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
return EINVAL;
}
mutex->prioceiling = attr->prioceiling;
mutex->mtxopts.options.protocol = attr->protocol;
mutex->mtxopts.options.policy = attr->policy;
mutex->mtxopts.options.type = attr->type;
mutex->mtxopts.options.pshared = attr->pshared;
} else {
switch (static_type) {
case 1:
mutex->mtxopts.options.type = PTHREAD_MUTEX_ERRORCHECK;
break;
case 2:
mutex->mtxopts.options.type = PTHREAD_MUTEX_RECURSIVE;
break;
case 3:
case 7:
mutex->mtxopts.options.type = PTHREAD_MUTEX_DEFAULT;
break;
default:
return EINVAL;
}
mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
mutex->mtxopts.options.protocol = _PTHREAD_DEFAULT_PROTOCOL;
if (static_type != 3) {
mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
} else {
mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
}
mutex->mtxopts.options.pshared = _PTHREAD_DEFAULT_PSHARED;
}
mutex->priority = 0;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
#if PTHREAD_MUTEX_INIT_UNUSED
if ((uint32_t*)tidaddr != mutex->m_tid) {
mutex->mtxopts.options.misalign = 1;
__builtin_memset(mutex->m_tid, 0xff, sizeof(mutex->m_tid));
}
__builtin_memset(mutex->m_mis, 0xff, sizeof(mutex->m_mis));
#endif // PTHREAD_MUTEX_INIT_UNUSED
*tidaddr = 0;
*seqaddr = 0;
long sig = _PTHREAD_MUTEX_SIG;
if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL &&
mutex->mtxopts.options.policy == _PTHREAD_MUTEX_POLICY_FAIRSHARE) {
sig = _PTHREAD_MUTEX_SIG_fast;
}
#if PTHREAD_MUTEX_INIT_UNUSED
uint32_t sig32 = (uint32_t)sig;
#if defined(__LP64__)
uintptr_t guard = ~(uintptr_t)mutex; __builtin_memcpy(mutex->_reserved, &guard, sizeof(guard));
mutex->_reserved[2] = sig32;
mutex->_reserved[3] = sig32;
mutex->_pad = sig32;
#else
mutex->_reserved[0] = sig32;
#endif
#endif // PTHREAD_MUTEX_INIT_UNUSED
#if defined(__LP64__)
uint32_t *sig32_ptr = (uint32_t*)&mutex->sig;
uint32_t *sig32_val = (uint32_t*)&sig;
*(sig32_ptr+1) = *(sig32_val+1);
os_atomic_store(sig32_ptr, *sig32_val, release);
#else
os_atomic_store2o(mutex, sig, sig, release);
#endif
return 0;
}
int
pthread_mutex_destroy(pthread_mutex_t *omutex)
{
_pthread_mutex *mutex = (_pthread_mutex *)omutex;
int res = EINVAL;
LOCK(mutex->lock);
if (_pthread_mutex_check_signature(mutex)) {
uint32_t lgenval, ugenval;
uint64_t oldval64;
volatile uint64_t *seqaddr;
MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
volatile uint64_t *tidaddr;
MUTEX_GETTID_ADDR(mutex, &tidaddr);
oldval64 = *seqaddr;
lgenval = (uint32_t)oldval64;
ugenval = (uint32_t)(oldval64 >> 32);
if ((*tidaddr == (uint64_t)0) &&
((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK))) {
mutex->sig = _PTHREAD_NO_SIG;
res = 0;
} else {
res = EBUSY;
}
} else if (_pthread_mutex_check_signature_init(mutex)) {
mutex->sig = _PTHREAD_NO_SIG;
res = 0;
}
UNLOCK(mutex->lock);
return res;
}
#endif // OS_UP_VARIANT_ONLY
#endif
#ifndef OS_UP_VARIANT_ONLY
int
pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
{
#if __DARWIN_UNIX03
if (__unix_conforming == 0) {
__unix_conforming = 1;
}
if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) {
return EINVAL;
}
#endif
attr->sig = _PTHREAD_NO_SIG;
return 0;
}
#endif // OS_UP_VARIANT_ONLY