#include "pthread_internals.h"
#include <stdio.h>
extern int __unix_conforming;
#ifdef PLOCKSTAT
#include "plockstat.h"
#else
#define PLOCKSTAT_RW_ERROR(x, y, z)
#define PLOCKSTAT_RW_BLOCK(x, y)
#define PLOCKSTAT_RW_BLOCKED(x, y, z)
#define PLOCKSTAT_RW_ACQUIRE(x, y)
#define PLOCKSTAT_RW_RELEASE(x, y)
#endif
#define READ_LOCK_PLOCKSTAT 0
#define WRITE_LOCK_PLOCKSTAT 1
#define BLOCK_FAIL_PLOCKSTAT 0
#define BLOCK_SUCCESS_PLOCKSTAT 1
#define MAX_READ_LOCKS (INT_MAX - 1)
#ifndef BUILDING_VARIANT
__private_extern__ int usenew_impl = 1;
#else
extern int usenew_impl;
#endif
extern int PR_5243343_flag;
#if defined(__LP64__)
#define RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr) \
{ \
if (rwlock->misalign != 0) { \
lcntaddr = &rwlock->rw_seq[1]; \
seqaddr = &rwlock->rw_seq[2]; \
ucntaddr = &rwlock->rw_seq[3]; \
} else { \
lcntaddr = &rwlock->rw_seq[0]; \
seqaddr = &rwlock->rw_seq[1]; \
ucntaddr = &rwlock->rw_seq[2]; \
} \
}
#else
#define RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr) \
{ \
if (rwlock->misalign != 0) { \
lcntaddr = &rwlock->rw_seq[1]; \
seqaddr = &rwlock->rw_seq[2]; \
ucntaddr = &rwlock->rw_seq[3]; \
} else { \
lcntaddr = &rwlock->rw_seq[0]; \
seqaddr = &rwlock->rw_seq[1]; \
ucntaddr = &rwlock->rw_seq[2]; \
} \
}
#endif
__private_extern__ int __pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr);
#define _KSYN_TRACE_ 0
#if _KSYN_TRACE_
#include <sys/sysctl.h>
#ifndef BUILDING_VARIANT
static void set_enable(int);
#endif
#define DBG_FUNC_START 1
#define DBG_FUNC_END 2
#define DBG_FUNC_NONE 0
int __kdebug_trace(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t);
#define _KSYN_TRACE_RW_RDLOCK 0x9000080
#define _KSYN_TRACE_RW_WRLOCK 0x9000084
#define _KSYN_TRACE_RW_UNLOCK 0x9000088
#define _KSYN_TRACE_RW_UNACT1 0x900808c
#define _KSYN_TRACE_RW_UNACT2 0x9008090
#define _KSYN_TRACE_RW_UNACTK 0x9008094
#define _KSYN_TRACE_RW_UNACTE 0x9008098
#define _KSYN_TRACE_RW_UNACTR 0x900809c
#define _KSYN_TRACE_RW_TOOMANY 0x90080a0
#define _KSYN_TRACE_RW_TRYWRLOCK 0x90080a4
#define _KSYN_TRACE_RW_TRYRDLOCK 0x90080a8
#endif
__private_extern__ void rwlock_action_onreturn(pthread_rwlock_t * rwlock, uint32_t updateval);
__private_extern__ int rw_diffgenseq(uint32_t x, uint32_t y);
#ifndef BUILDING_VARIANT
static uint32_t modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits);
int
pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
{
attr->sig = _PTHREAD_RWLOCK_ATTR_SIG;
attr->pshared = _PTHREAD_DEFAULT_PSHARED;
return (0);
}
int
pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
{
attr->sig = _PTHREAD_NO_SIG;
attr->pshared = 0;
return (0);
}
int
pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr,
int *pshared)
{
if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG)
{
*pshared = (int)attr->pshared;
return (0);
} else
{
return (EINVAL);
}
}
int
pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared)
{
if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG)
{
#if __DARWIN_UNIX03
if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED))
#else
if ( pshared == PTHREAD_PROCESS_PRIVATE)
#endif
{
attr->pshared = pshared ;
return (0);
} else
{
return (EINVAL);
}
} else
{
return (EINVAL);
}
}
__private_extern__ int
__pthread_rwlock_init(pthread_rwlock_t * orwlock, const pthread_rwlockattr_t *attr)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
if ((attr != NULL) && (attr->pshared == PTHREAD_PROCESS_SHARED)) {
rwlock->pshared = PTHREAD_PROCESS_SHARED;
rwlock->rw_flags = PTHRW_KERN_PROCESS_SHARED;
} else {
rwlock->pshared = _PTHREAD_DEFAULT_PSHARED;
rwlock->rw_flags = PTHRW_KERN_PROCESS_PRIVATE;
}
if (((uintptr_t)rwlock & 0x07) != 0) {
rwlock->misalign = 1;
#if defined(__LP64__)
rwlock->rw_lcntaddr = &rwlock->rw_seq[1];
rwlock->rw_seqaddr = &rwlock->rw_seq[2];
rwlock->rw_ucntaddr = &rwlock->rw_seq[3];
rwlock->rw_seq[1]= PTHRW_RWLOCK_INIT;
rwlock->rw_seq[2]= PTHRW_RWS_INIT;
rwlock->rw_seq[3]= 0;
#else
rwlock->rw_lcntaddr = &rwlock->rw_seq[1];
rwlock->rw_seqaddr = &rwlock->rw_seq[2];
rwlock->rw_ucntaddr = &rwlock->rw_seq[3];
rwlock->rw_seq[1]= PTHRW_RWLOCK_INIT;
rwlock->rw_seq[2]= PTHRW_RWS_INIT;
rwlock->rw_seq[3]= 0;
#endif
} else {
rwlock->misalign = 0;
#if defined(__LP64__)
rwlock->rw_lcntaddr = &rwlock->rw_seq[0];
rwlock->rw_seqaddr = &rwlock->rw_seq[1];
rwlock->rw_ucntaddr = &rwlock->rw_seq[2];
rwlock->rw_seq[0]= PTHRW_RWLOCK_INIT;
rwlock->rw_seq[1]= PTHRW_RWS_INIT;
rwlock->rw_seq[2]= 0;
#else
rwlock->rw_lcntaddr = &rwlock->rw_seq[0];
rwlock->rw_seqaddr = &rwlock->rw_seq[1];
rwlock->rw_ucntaddr = &rwlock->rw_seq[2];
rwlock->rw_seq[0]= PTHRW_RWLOCK_INIT;
rwlock->rw_seq[1]= PTHRW_RWS_INIT;
rwlock->rw_seq[2]= 0;
#endif
}
rwlock->reserv = 0;
rwlock->rw_owner = NULL;
#if defined(__LP64__)
memset(rwlock->rfu, 0, PTHRW_RFU_64BIT);
#else
memset(rwlock->rfu, 0, PTHRW_RFU_32BIT);
#endif
rwlock->sig = _PTHREAD_RWLOCK_SIG;
return(0);
}
#if _KSYN_TRACE_
static void
set_enable(int val)
{
int mib[6];
size_t needed = 0;
mib[0] = CTL_KERN;
mib[1] = KERN_KDEBUG;
mib[2] = KERN_KDENABLE;
mib[3] = val;
mib[4] = 0;
mib[5] = 0;
(void)sysctl(mib, 4, NULL, &needed, NULL, 0);
}
#endif
static uint32_t
modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits)
{
uint32_t lval = lgenval & PTHRW_BIT_MASK;
uint32_t uval = updateval & PTHRW_BIT_MASK;
uint32_t rval, nlval;
nlval = (lval | uval) & ~(PTH_RWL_MBIT);
if ((uval & PTH_RWL_LBIT) != 0)
nlval &= ~PTH_RWL_KBIT;
else if (((uval & PTH_RWL_KBIT) == 0) && ((lval & PTH_RWL_WBIT) == 0))
nlval &= ~PTH_RWL_KBIT;
if (savebits !=0 ) {
if (((savebits & PTH_RWS_WSVBIT) != 0) && ((nlval & PTH_RWL_WBIT) == 0) &&
((nlval & PTH_RWL_EBIT) == 0)) {
if ((nlval & PTH_RWL_LBIT) == 0)
nlval |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
else
nlval |= PTH_RWL_WBIT;
}
if (((savebits & PTH_RWS_YSVBIT) != 0) && ((nlval & PTH_RWL_YBIT) == 0) &&
((nlval & PTH_RWL_EBIT) == 0)) {
nlval |= PTH_RWL_YBIT;
}
if (((savebits & PTH_RWS_USVBIT) != 0) && ((nlval & PTH_RWL_EBIT) == 0)) {
if ((nlval & PTH_RWL_LBIT) == 0)
nlval |= (PTH_RWL_UBIT | PTH_RWL_KBIT);
else
nlval |= PTH_RWL_UBIT;
}
}
rval = (lgenval & PTHRW_COUNT_MASK) | nlval;
return(rval);
}
__private_extern__ void
rwlock_action_onreturn(pthread_rwlock_t * orwlock, uint32_t updateval)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lcntval, rw_seq, newval = 0, newsval, lval, uval;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t oldval64, newval64;
int setbits = 0;
int overlap = 0;
uint32_t savebits = 0;
int isoverlap = 0;
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
seqaddr = rwlock->rw_seqaddr;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_START, updateval, 0, 0, 0, 0);
#endif
isoverlap = updateval & PTH_RWL_MBIT;
loop:
setbits = 0;
lcntval = *lcntaddr;
rw_seq = *seqaddr;
savebits = 0;
if (isoverlap != 0) {
setbits = 1;
overlap = 1;
newsval = rw_seq + (updateval & PTHRW_COUNT_MASK);
if ((newsval & PTHRW_RWS_SAVEMASK) != 0) {
savebits = newsval & PTHRW_RWS_SAVEMASK;
newsval &= ~PTHRW_RWS_SAVEMASK;
}
} else {
if (is_rws_setunlockinit(rw_seq) != 0) {
setbits = 1;
newsval = (rw_seq & PTHRW_COUNT_MASK) + (updateval & PTHRW_COUNT_MASK);
if ((rw_seq & PTHRW_RWS_SAVEMASK) != 0) {
savebits = rw_seq & PTHRW_RWS_SAVEMASK;
newsval &= ~PTHRW_RWS_SAVEMASK;
}
} else {
newval = lcntval;
newsval = rw_seq;
}
}
if (setbits != 0) {
newval = modbits(lcntval, updateval, savebits);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_NONE, rw_seq, newsval, 0xeeeeeeee, updateval, 0);
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_NONE, lcntval, newval, 0xeeeeeeee, updateval, 0);
#endif
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
lval = lcntval & PTHRW_BIT_MASK;
uval = updateval & PTHRW_BIT_MASK;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_END, rw_seq, newsval, 0xffffffff, 0, 0);
#endif
return;
}
__private_extern__ int
rw_diffgenseq(uint32_t x, uint32_t y)
{
uint32_t lx = (x & PTHRW_COUNT_MASK);
uint32_t ly = (y &PTHRW_COUNT_MASK);
if (lx > ly) {
return(lx-ly);
} else {
return((PTHRW_MAX_READERS - y) + lx + PTHRW_INC);
}
}
#ifdef NOTYET
static int pthread_rwlock_upgrade_internal(pthread_rwlock_t * orwlock, int trylock);
int
pthread_rwlock_longrdlock_np(pthread_rwlock_t * orwlock)
{
pthread_t self;
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval;
int error = 0, retry_count = 0;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
if (can_rwl_longreadinuser(lcntval))
goto gotlock;
#if __DARWIN_UNIX03
if (is_rwl_ebit_set(lcntval)) {
self = pthread_self();
if(rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
newval = (lcntval + PTHRW_INC);
newsval = rw_seq;
if (is_rws_setseq(rw_seq)) {
newsval &= PTHRW_SW_Reset_BIT_MASK;
newsval |= (newval & PTHRW_COUNT_MASK);
}
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)(newsval)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
kblock:
updateval = __psynch_rw_longrdlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto kblock;
if (error == 0) {
rwlock_action_onreturn(orwlock, updateval);
if ( is_rwl_lbit_clear(updateval)) {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("yieldwrlock from kernel without EBit %x: tid %x\n", updateval, (uint32_t)myid);
}
goto successout;
} else {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("yieldwrlock from kernel with unknown error %x: tid %x\n", updateval, (uint32_t)myid);
goto out;
}
gotlock:
if (rw_diffgenseq(lcntval, ucntval) >= PTHRW_MAX_READERS) {
retry_count++;
if (retry_count > 1024) {
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TOOMANY | DBG_FUNC_NONE, (uint32_t)rwlock, 0XEEEEEEEE, lcntval, ucntval, 0);
#endif
error = EAGAIN;
goto out;
} else {
sched_yield();
goto loop;
}
}
newval = (lcntval + PTHRW_INC) | PTH_RWL_LBIT;
newsval = (rw_seq + PTHRW_INC);
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
successout:
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
return(0);
out:
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
int
pthread_rwlock_yieldwrlock_np(pthread_rwlock_t * orwlock)
{
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval;
int error = 0;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
#if __DARWIN_UNIX03
if (is_rwl_ebit_set(lcntval)) {
if (rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
if (lcntval == PTHRW_RWL_INIT) {
lcntval = PTHRW_RWL_INIT;
newval = PTHRW_RWL_INIT | PTHRW_INC | PTH_RWL_KBIT| PTH_RWL_EBIT;
newsval = rw_seq + PTHRW_INC;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) == TRUE) {
goto gotit;
} else
goto loop;
}
newval = (lcntval + PTHRW_INC)| PTH_RWL_YBIT;
newsval = rw_seq;
if (is_rws_setseq(rw_seq)) {
newsval &= PTHRW_SW_Reset_BIT_MASK;
newsval |= (newval & PTHRW_COUNT_MASK);
}
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)(newsval)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
PLOCKSTAT_RW_BLOCK(orwlock, WRITE_LOCK_PLOCKSTAT);
retry:
updateval = __psynch_rw_yieldwrlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
PLOCKSTAT_RW_BLOCKED(orwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
if (error != 0) {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("yieldwrlock from kernel with unknown error %x: tid %x\n", updateval, (uint32_t)myid);
}
out:
if (error == 0) {
gotit:
rwlock_action_onreturn(orwlock, updateval);
if ( is_rwl_ebit_clear(updateval)) {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("yieldwrlock from kernel without EBit %x: tid %x\n", updateval, (uint32_t)myid);
}
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
return(0);
} else {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
}
int
pthread_rwlock_downgrade_np(pthread_rwlock_t * orwlock)
{
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval;
int error = 0, haswbit = 0, hasubit = 0, hasybit = 0;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
if ((is_rwl_ebit_set(lcntval )== 0) || (rwlock->rw_owner != self)) {
return(EINVAL);
}
if ((lcntval & PTHRW_COUNT_MASK) == (ucntval + PTHRW_INC)) {
#if 0
if (is_rwl_wbit_set(lcntval) != 0) {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("downgrade in user mode but W bit set %x: tid %x\n", lcntval, (uint32_t)myid);
}
#endif
newval = lcntval & ~(PTH_RWL_EBIT | PTH_RWL_KBIT);
newsval = rw_seq;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) == TRUE) {
#if __DARWIN_UNIX03
rwlock->rw_owner = (pthread_t)0;
#endif
return(0);
} else
goto loop;
} else {
haswbit = lcntval & PTH_RWL_WBIT;
hasubit = lcntval & PTH_RWL_UBIT;
hasybit = lcntval & PTH_RWL_YBIT;
newval = (lcntval & PTHRW_COUNT_MASK) | PTH_RWL_KBIT;
newsval = rw_seq | PTH_RWS_IBIT;
if (haswbit != 0)
newsval |= PTH_RWS_WSVBIT;
if (hasubit != 0)
newsval |= PTH_RWS_USVBIT;
if (hasybit != 0)
newsval |= PTH_RWS_YSVBIT;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
#if __DARWIN_UNIX03
rwlock->rw_owner = 0;
#endif
retry:
updateval = __psynch_rw_downgrade(orwlock, newval, ucntval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
if (error == 0) {
rwlock_action_onreturn(orwlock, updateval);
return(0);
} else {
#if _KSYN_TRACE_
set_enable(1);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("downgrade from kernel with unknown error %x with tid %x\n", updateval, (uint32_t)myid);
}
}
return(EINVAL);
}
int
pthread_rwlock_upgrade_np(pthread_rwlock_t * orwlock)
{
return(pthread_rwlock_upgrade_internal(orwlock, 0));
}
int
pthread_rwlock_tryupgrade_np(pthread_rwlock_t *orwlock)
{
return(pthread_rwlock_upgrade_internal(orwlock, 1));
}
static int
pthread_rwlock_upgrade_internal(pthread_rwlock_t * orwlock, int trylock)
{
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval;
int error = 0, flags ;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
if (is_rwl_eubit_set(lcntval) !=0) {
return(EBUSY);
}
newval = (lcntval | (PTH_RWL_UBIT | PTH_RWL_KBIT));
newsval = rw_seq;
#if 0
if (is_rws_setseq(rw_seq)) {
newsval &= PTHRW_SW_Reset_BIT_MASK;
newsval |= (newval & PTHRW_COUNT_MASK);
}
#endif
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)(newsval)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
flags = rwlock->rw_flags;
if (trylock != 0) {
flags |= _PTHREAD_RWLOCK_UPGRADE_TRY;
}
retry:
updateval = __psynch_rw_upgrade(orwlock, newval, ucntval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
if (error == 0) {
rwlock_action_onreturn(orwlock, updateval);
if ( is_rwl_ebit_clear(updateval)) {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("upgrade from kernel without EBit %x: tid %x\n", updateval, (uint32_t)myid);
}
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
return(0);
} else {
if (trylock != 0) {
return (EBUSY);
}
}
return(error);
}
int
pthread_rwlock_held_np(pthread_rwlock_t * orwlock)
{
uint32_t lcntval, ucntval, rw_seq;
int error = 0;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
return(0);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
return(-1);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
if ((lcntval & PTHRW_COUNT_MASK) == (ucntval & PTHRW_COUNT_MASK))
return(0);
return(1);
}
int
pthread_rwlock_rdheld_np(pthread_rwlock_t * orwlock)
{
uint32_t lcntval, ucntval, rw_seq;
int error = 0;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
return(0);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
return(-1);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
if ((lcntval & PTHRW_COUNT_MASK) == (ucntval & PTHRW_COUNT_MASK))
return(0);
if (is_rwl_ebit_set(lcntval) !=0) {
return(0);
}
return(1);
}
int
pthread_rwlock_wrheld_np(pthread_rwlock_t * orwlock)
{
uint32_t lcntval, ucntval, rw_seq;
pthread_t self = pthread_self();
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
int error = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
return(0);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
return(-1);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
if ((is_rwl_ebit_set(lcntval)) && (rwlock->rw_owner == self)) {
return(1);
}
return(0);
}
#endif
#endif
int
pthread_rwlock_destroy(pthread_rwlock_t *orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
#if __DARWIN_UNIX03
uint32_t rw_lcnt, rw_ucnt;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
#endif
if (rwlock->sig != _PTHREAD_RWLOCK_SIG && rwlock->sig != _PTHREAD_RWLOCK_SIG_init)
return(EINVAL);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG) {
#if __DARWIN_UNIX03
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
rw_lcnt = *lcntaddr;
rw_ucnt = *ucntaddr;
if((rw_lcnt & PTHRW_COUNT_MASK) != rw_ucnt)
return(EBUSY);
#endif
rwlock->sig = _PTHREAD_NO_SIG;
return(0);
} else if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
rwlock->sig = _PTHREAD_NO_SIG;
return(0);
} else
return(EINVAL);
}
int
pthread_rwlock_init(pthread_rwlock_t * orwlock, const pthread_rwlockattr_t *attr)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
#if __DARWIN_UNIX03
uint32_t rw_lcnt, rw_ucnt;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
#endif
#if __DARWIN_UNIX03
if (attr && (attr->sig != _PTHREAD_RWLOCK_ATTR_SIG)) {
return(EINVAL);
}
if (rwlock->sig == _PTHREAD_RWLOCK_SIG) {
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
rw_lcnt = *lcntaddr;
rw_ucnt = *ucntaddr;
if ((rw_lcnt & PTHRW_COUNT_MASK) != rw_ucnt)
return(EBUSY);
}
#endif
LOCK_INIT(rwlock->lock);
return(__pthread_rwlock_init(orwlock, attr));
}
int
pthread_rwlock_rdlock(pthread_rwlock_t * orwlock)
{
#if __DARWIN_UNIX03
pthread_t self;
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval;
int error = 0, retry_count = 0;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lcntval, (ucntval | 0xee), rw_seq, 0);
#endif
if (can_rwl_readinuser(lcntval))
goto gotlock;
#if __DARWIN_UNIX03
if (is_rwl_ebit_set(lcntval)) {
self = pthread_self();
if(rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
newval = (lcntval + PTHRW_INC) & PTH_RWLOCK_RESET_RBIT;
newsval = rw_seq;
if (is_rws_setseq(rw_seq)) {
newsval &= PTHRW_SW_Reset_BIT_MASK;
newsval |= (newval & PTHRW_COUNT_MASK);
}
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
PLOCKSTAT_RW_BLOCK(orwlock, READ_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lcntval, newval, newsval, 0);
#endif
retry:
updateval = __psynch_rw_rdlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
if (error == 0) {
rwlock_action_onreturn(orwlock, updateval);
PLOCKSTAT_RW_BLOCKED(orwlock, READ_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
return(0);
} else {
PLOCKSTAT_RW_BLOCKED(orwlock, READ_LOCK_PLOCKSTAT, BLOCK_FAIL_PLOCKSTAT);
#if _KSYN_TRACE_
set_enable(1);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("rdlock from kernel with unknown error %x with tid %x\n", updateval, (uint32_t)myid);
goto out;
}
gotlock:
if (rw_diffgenseq(lcntval, ucntval) >= PTHRW_MAX_READERS) {
retry_count++;
if (retry_count > 1024) {
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TOOMANY | DBG_FUNC_NONE, (uint32_t)rwlock, 0XEEEEEEEE, lcntval, ucntval, 0);
#endif
error = EAGAIN;
goto out;
} else {
sched_yield();
goto loop;
}
}
newval = (lcntval + PTHRW_INC) & PTH_RWLOCK_RESET_RBIT;
newsval = (rw_seq + PTHRW_INC);
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lcntval, newval, 0);
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, 0, 0, 0);
#endif
return(0);
out:
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(error);
}
int
pthread_rwlock_tryrdlock(pthread_rwlock_t * orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lcntval, ucntval, rw_seq, newval, newsval;
int error = 0, retry_count = 0;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TRYRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lcntval, ucntval, rw_seq, 0);
#endif
if (can_rwl_readinuser(lcntval))
goto gotlock;
error = EBUSY;
goto out;
gotlock:
if (rw_diffgenseq(lcntval, ucntval) >= PTHRW_MAX_READERS) {
retry_count++;
if (retry_count > 1024) {
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TOOMANY | DBG_FUNC_NONE, (uint32_t)rwlock, 0XEEEEEEEE, lcntval, ucntval, 0);
#endif
error = EAGAIN;
goto out;
} else {
sched_yield();
goto loop;
}
}
newval = (lcntval + PTHRW_INC) & PTH_RWLOCK_RESET_RBIT;
newsval = (rw_seq + PTHRW_INC);
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TRYRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lcntval, newval, 0);
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
return(0);
out:
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TRYRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, error, 0, 0);
#endif
return(error);
}
int
pthread_rwlock_trywrlock(pthread_rwlock_t * orwlock)
{
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lcntval, rw_seq, newval, newsval;
int error = 0, gotlock = 0;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TRYWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
loop:
lcntval = *lcntaddr;
rw_seq = *seqaddr;
if ((lcntval & PTH_RWL_RBIT) != 0) {
newval = ((lcntval + PTHRW_INC) & PTHRW_COUNT_MASK) | PTH_RWL_IBIT | PTH_RWL_KBIT| PTH_RWL_EBIT;
newsval = rw_seq + PTHRW_INC;
gotlock = 1;
} else
gotlock = 0;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
if (gotlock != 0) {
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
} else
newval64 = oldval64;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) {
goto loop;
}
if (gotlock == 1) {
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TRYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
return(0);
} else {
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_TRYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, EBUSY, 0, 0);
#endif
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EBUSY);
return(EBUSY);
}
}
int
pthread_rwlock_wrlock(pthread_rwlock_t * orwlock)
{
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval;
int error = 0, gotlock = 0;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lcntval, ucntval, rw_seq, 0);
#endif
#if __DARWIN_UNIX03
if (is_rwl_ebit_set(lcntval)) {
if(rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
if ((lcntval & PTH_RWL_RBIT) != 0) {
newval = ((lcntval + PTHRW_INC) & PTHRW_COUNT_MASK) | PTH_RWL_IBIT | PTH_RWL_KBIT| PTH_RWL_EBIT;
newsval = rw_seq + PTHRW_INC;
gotlock = 1;
} else {
if (is_rwl_lbit_set(lcntval))
newval = (lcntval + PTHRW_INC)| PTH_RWL_WBIT;
else
newval = (lcntval + PTHRW_INC) | PTH_RWL_KBIT| PTH_RWL_WBIT;
newsval = rw_seq;
if (is_rws_setseq(rw_seq)) {
newsval &= PTHRW_SW_Reset_BIT_MASK;
newsval |= (newval & PTHRW_COUNT_MASK);
}
gotlock = 0;
}
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lcntval, newval, 0);
#endif
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto loop;
if (gotlock != 0)
goto gotit;
PLOCKSTAT_RW_BLOCK(orwlock, WRITE_LOCK_PLOCKSTAT);
retry:
updateval = __psynch_rw_wrlock(orwlock, newval, ucntval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR) {
goto retry;
}
if (error != 0) {
#if _KSYN_TRACE_
set_enable(2);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("wrlock from kernel with unknown error %x: tid %x\n", updateval, (uint32_t)myid);
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x33333333, newval, updateval, 0);
#endif
PLOCKSTAT_RW_BLOCKED(orwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
if (error == 0) {
rwlock_action_onreturn(orwlock, updateval);
gotit:
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(0);
}
#if __DARWIN_UNIX03
out:
#endif
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(error);
}
int
pthread_rwlock_unlock(pthread_rwlock_t * orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lcntval, ucntval, rw_seq, newval, newsval, updateval, ulval;
int error = 0, wrlock = 0, haswbit = 0, hasubit = 0, hasybit = 0;
uint64_t oldval64, newval64;
volatile uint32_t * lcntaddr, *ucntaddr, *seqaddr;
uint64_t myid = 0;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
LOCK(rwlock->lock);
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = __pthread_rwlock_init(orwlock, NULL)) != 0) {
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, wrlock, error);
return(error);
}
} else if (rwlock->sig != _PTHREAD_RWLOCK_SIG){
UNLOCK(rwlock->lock);
PLOCKSTAT_RW_ERROR(orwlock, wrlock, EINVAL);
return(EINVAL);
}
UNLOCK(rwlock->lock);
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lcntaddr, ucntaddr, seqaddr);
} else {
lcntaddr = rwlock->rw_lcntaddr;
ucntaddr = rwlock->rw_ucntaddr;
seqaddr = rwlock->rw_seqaddr;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
loop:
lcntval = *lcntaddr;
ucntval = *ucntaddr;
rw_seq = *seqaddr;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x51515151, lcntval, ucntval, 0);
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x51515151, rw_seq, 0, 0);
#endif
if ((lcntval & PTH_RWL_RBIT) != 0) {
newval = lcntval ;
newsval = rw_seq;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) == TRUE) {
error = EINVAL;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x1a1b1c1d, lcntval, ucntval, 0);
#endif
goto succout;
} else
goto loop;
}
if (is_rwl_ebit_set(lcntval)) {
wrlock = 1;
#if __DARWIN_UNIX03
rwlock->rw_owner = (pthread_t)0;
#endif
}
ulval = (ucntval + PTHRW_INC);
if (OSAtomicCompareAndSwap32Barrier(ucntval, ulval, (volatile int32_t *)ucntaddr) != TRUE)
goto loop;
lp11:
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = oldval64;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) {
lcntval = *lcntaddr;
rw_seq = *seqaddr;
goto lp11;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xd1d2d3d4, lcntval, rw_seq, 0);
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xd1d2d3d4, ulval, 0, 0);
#endif
if((lcntval & PTHRW_COUNT_MASK) == (ulval & PTHRW_COUNT_MASK)) {
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xbbbbbbbb, lcntval, ucntval, 0);
#endif
newval = (lcntval & PTHRW_COUNT_MASK)| PTHRW_RWLOCK_INIT;
newsval = (lcntval & PTHRW_COUNT_MASK)| PTHRW_RWS_INIT;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE) {
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xcccccccc, 0, 0, 0);
#endif
lcntval = *lcntaddr;
rw_seq = *seqaddr;
goto lp11;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xdddddddd, lcntval, ucntval, 0);
#endif
goto succout;
}
if ((lcntval & (PTH_RWL_EBIT | PTH_RWL_WBIT | PTH_RWL_YBIT | PTH_RWL_KBIT)) == 0) {
goto succout;
}
if ((ulval + PTHRW_INC) != (rw_seq & PTHRW_COUNT_MASK)) {
if ((lcntval & PTH_RWL_UBIT) != 0) {
if ((ulval + PTHRW_INC + PTHRW_INC) != (rw_seq & PTHRW_COUNT_MASK))
goto succout;
} else
goto succout;
}
haswbit = lcntval & PTH_RWL_WBIT;
hasubit = lcntval & PTH_RWL_UBIT;
hasybit = lcntval & PTH_RWL_YBIT;
newval = (lcntval & PTHRW_COUNT_MASK) | PTH_RWL_KBIT;
newsval = rw_seq | PTH_RWS_IBIT;
if (haswbit != 0)
newsval |= PTH_RWS_WSVBIT;
if (hasubit != 0)
newsval |= PTH_RWS_USVBIT;
if (hasybit != 0)
newsval |= PTH_RWS_YSVBIT;
oldval64 = (((uint64_t)rw_seq) << 32);
oldval64 |= lcntval;
newval64 = (((uint64_t)newsval) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64Barrier(oldval64, newval64, (volatile int64_t *)lcntaddr) != TRUE)
goto lp11;
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555511, 1, ulval, 0);
#endif
updateval = __psynch_rw_unlock(orwlock, lcntval, ulval, newsval, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if(error != 0) {
if(error != EINTR) {
#if _KSYN_TRACE_
set_enable(4);
#endif
(void)pthread_threadid_np(pthread_self(), &myid);
LIBC_ABORT("rwunlock from kernel with unknown error %x: tid %x\n", error, (uint32_t)myid);
goto succout;
}
error = 0;
}
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555522, 3, lcntval, 0);
#endif
succout:
PLOCKSTAT_RW_RELEASE(orwlock, wrlock);
#if _KSYN_TRACE_
if (__pthread_lock_debug != 0)
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(0);
}