#include "pthread_internals.h"
#include <stdio.h>
extern int __unix_conforming;
#ifdef PLOCKSTAT
#include "plockstat.h"
#else
#define PLOCKSTAT_RW_ERROR(x, y, z)
#define PLOCKSTAT_RW_BLOCK(x, y)
#define PLOCKSTAT_RW_BLOCKED(x, y, z)
#define PLOCKSTAT_RW_ACQUIRE(x, y)
#define PLOCKSTAT_RW_RELEASE(x, y)
#endif
#define READ_LOCK_PLOCKSTAT 0
#define WRITE_LOCK_PLOCKSTAT 1
#define BLOCK_FAIL_PLOCKSTAT 0
#define BLOCK_SUCCESS_PLOCKSTAT 1
#define MAX_READ_LOCKS (INT_MAX - 1)
#if defined(__i386__) || defined(__x86_64__)
#ifndef BUILDING_VARIANT
int usenew_impl = 0;
#else
extern int usenew_impl;
#endif
#if defined(__LP64__)
#define RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr) \
{ \
if (rwlock->misalign != 0) { \
lseqaddr = &rwlock->rw_seq[1]; \
wcaddr = &rwlock->rw_seq[2]; \
useqaddr = &rwlock->rw_seq[3]; \
} else { \
lseqaddr = &rwlock->rw_seq[0]; \
wcaddr = &rwlock->rw_seq[1]; \
useqaddr = &rwlock->rw_seq[2]; \
} \
}
#else
#define RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr) \
{ \
if (rwlock->misalign != 0) { \
lseqaddr = &rwlock->rw_seq[0]; \
wcaddr = &rwlock->rw_seq[1]; \
useqaddr = &rwlock->rw_seq[2]; \
}else { \
lseqaddr = &rwlock->rw_seq[1]; \
wcaddr = &rwlock->rw_seq[2]; \
useqaddr = &rwlock->rw_seq[3]; \
} \
}
#endif
int _new_pthread_rwlock_destroy(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr);
int _new_pthread_rwlock_rdlock(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_longrdlock_np(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_wrlock(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_yieldwrlock_np(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_unlock(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_downgrade_np(pthread_rwlock_t *rwlock);
int _new_pthread_rwlock_upgrade_np(pthread_rwlock_t *rwlock);
#define _KSYN_TRACE_ 0
#if _KSYN_TRACE_
#define DBG_FUNC_START 1
#define DBG_FUNC_END 2
#define DBG_FUNC_NONE 0
int __kdebug_trace(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t);
#define _KSYN_TRACE_RW_RDLOCK 0x9000080
#define _KSYN_TRACE_RW_WRLOCK 0x9000084
#define _KSYN_TRACE_RW_UNLOCK 0x9000088
#define _KSYN_TRACE_RW_UNACT1 0x900808c
#define _KSYN_TRACE_RW_UNACT2 0x9008090
#define _KSYN_TRACE_RW_UNACTK 0x9008094
#define _KSYN_TRACE_RW_UNACTE 0x9008098
#endif
#endif
#ifndef BUILDING_VARIANT
#if defined(__i386__) || defined(__x86_64__)
static int rwlock_unlock_action_onread(pthread_rwlock_t * rwlock, uint32_t updateval);
static int rwlock_unlock_action1(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t updateval);
static int rwlock_unlock_action2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t updateval);
static uint32_t modbits(uint32_t lgenval, uint32_t updateval);
static int rwlock_unlock_action_k(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t updateval);
static int rwlock_exclusive_lockreturn(pthread_rwlock_t * rwlock, uint32_t updateval);
static int rw_diffgenseq(uint32_t x, uint32_t y);
#endif
int
pthread_rwlockattr_init(pthread_rwlockattr_t *attr)
{
attr->sig = _PTHREAD_RWLOCK_ATTR_SIG;
attr->pshared = _PTHREAD_DEFAULT_PSHARED;
return (0);
}
int
pthread_rwlockattr_destroy(pthread_rwlockattr_t *attr)
{
attr->sig = _PTHREAD_NO_SIG;
attr->pshared = 0;
return (0);
}
int
pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr,
int *pshared)
{
if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG)
{
*pshared = (int)attr->pshared;
return (0);
} else
{
return (EINVAL);
}
}
int
pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared)
{
if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG)
{
#if __DARWIN_UNIX03
if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED))
#else
if ( pshared == PTHREAD_PROCESS_PRIVATE)
#endif
{
attr->pshared = pshared ;
return (0);
} else
{
return (EINVAL);
}
} else
{
return (EINVAL);
}
}
#if defined(__i386__) || defined(__x86_64__)
int
_new_pthread_rwlock_destroy(pthread_rwlock_t *orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
#if __DARWIN_UNIX03
uint32_t rw_lseqcnt, rw_useqcnt;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
#endif
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
return(EINVAL);
} else {
#if __DARWIN_UNIX03
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
rw_lseqcnt = *lseqaddr;
rw_useqcnt = *useqaddr;
if((rw_lseqcnt & PTHRW_COUNT_MASK) != rw_useqcnt)
return(EBUSY);
#endif
rwlock->sig = _PTHREAD_NO_SIG;
return(0);
}
}
int
_new_pthread_rwlock_init(pthread_rwlock_t * orwlock, const pthread_rwlockattr_t *attr)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
#if __DARWIN_UNIX03
uint32_t rw_lseqcnt, rw_useqcnt;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
#endif
#if __DARWIN_UNIX03
if (attr && (attr->sig != _PTHREAD_RWLOCK_ATTR_SIG)) {
return(EINVAL);
}
if (rwlock->sig == _PTHREAD_RWLOCK_SIG) {
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
rw_lseqcnt = *lseqaddr;
rw_useqcnt = *useqaddr;
if ((rw_lseqcnt & PTHRW_COUNT_MASK) != rw_useqcnt)
return(EBUSY);
}
#endif
bzero(rwlock, sizeof(pthread_rwlock_t));
if ((attr != NULL) && (attr->pshared == PTHREAD_PROCESS_SHARED)) {
rwlock->pshared = PTHREAD_PROCESS_SHARED;
rwlock->rw_flags = PTHRW_KERN_PROCESS_SHARED;
} else {
rwlock->pshared = _PTHREAD_DEFAULT_PSHARED;
rwlock->rw_flags = PTHRW_KERN_PROCESS_PRIVATE;
}
if (((uintptr_t)rwlock & 0x07) != 0) {
rwlock->misalign = 1;
#if defined(__LP64__)
rwlock->rw_lseqaddr = &rwlock->rw_seq[1];
rwlock->rw_wcaddr = &rwlock->rw_seq[2];
rwlock->rw_useqaddr = &rwlock->rw_seq[3];
rwlock->rw_seq[1]= PTHRW_RW_INIT;
#else
rwlock->rw_lseqaddr = &rwlock->rw_seq[0];
rwlock->rw_wcaddr = &rwlock->rw_seq[1];
rwlock->rw_useqaddr = &rwlock->rw_seq[2];
rwlock->rw_seq[0]= PTHRW_RW_INIT;
#endif
} else {
rwlock->misalign = 0;
#if defined(__LP64__)
rwlock->rw_lseqaddr = &rwlock->rw_seq[0];
rwlock->rw_wcaddr = &rwlock->rw_seq[1];
rwlock->rw_useqaddr = &rwlock->rw_seq[2];
rwlock->rw_seq[0]= PTHRW_RW_INIT;
#else
rwlock->rw_lseqaddr = &rwlock->rw_seq[1];
rwlock->rw_wcaddr = &rwlock->rw_seq[2];
rwlock->rw_useqaddr = &rwlock->rw_seq[3];
rwlock->rw_seq[1]= PTHRW_RW_INIT;
#endif
}
rwlock->sig = _PTHREAD_RWLOCK_SIG;
return(0);
}
int
_new_pthread_rwlock_rdlock(pthread_rwlock_t * orwlock)
{
#if __DARWIN_UNIX03
pthread_t self;
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lgenval, ugenval, rw_wc, newval, updateval;
int error = 0, ret;
uint64_t oldval64, newval64;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgenval, newval, rw_wc, 0);
#endif
if (is_rw_lbit_set(lgenval))
goto gotlock;
if(is_rw_ewubit_clear(lgenval))
goto gotlock;
#if __DARWIN_UNIX03
if (is_rw_ebit_set(lgenval)) {
self = pthread_self();
if(rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
newval = (lgenval + PTHRW_INC);
oldval64 = (((uint64_t)rw_wc) << 32);
oldval64 |= lgenval;
newval64 = (((uint64_t)(rw_wc + 1)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lseqaddr) != TRUE)
goto loop;
PLOCKSTAT_RW_BLOCK(orwlock, READ_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lgenval, newval, rw_wc+1, 0);
#endif
retry:
updateval = __psynch_rw_rdlock(orwlock, (newval & ~PTHRW_RW_INIT), ugenval, rw_wc, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
OSAtomicDecrement32((volatile int32_t *)wcaddr);
if (error == 0) {
if ((updateval & PTHRW_RW_HUNLOCK) != 0) {
ret = rwlock_unlock_action_onread(orwlock, (updateval & ~PTHRW_RW_HUNLOCK));
if (ret != 0) {
LIBC_ABORT("rdlock_unlock handling failed");
}
}
PLOCKSTAT_RW_BLOCKED(orwlock, READ_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
return(0);
} else {
PLOCKSTAT_RW_BLOCKED(orwlock, READ_LOCK_PLOCKSTAT, BLOCK_FAIL_PLOCKSTAT);
goto out;
}
gotlock:
ugenval = *useqaddr;
if (rw_diffgenseq(lgenval, ugenval) >= PTHRW_MAX_READERS) {
error = EAGAIN;
goto out;
}
newval = (lgenval + PTHRW_INC);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lgenval, newval, 0);
#endif
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, 0, 0, 0);
#endif
return(0);
} else
goto loop;
out:
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_RDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(error);
}
int
_new_pthread_rwlock_tryrdlock(pthread_rwlock_t * orwlock)
{
uint32_t lgenval, newval, ugenval;
int error = 0;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
loop:
lgenval = *lseqaddr;
if (is_rw_lbit_set(lgenval))
goto gotlock;
if (is_rw_ewubit_clear(lgenval))
goto gotlock;
error = EBUSY;
goto out;
gotlock:
ugenval = *useqaddr;
if (rw_diffgenseq(lgenval, ugenval) >= PTHRW_MAX_READERS) {
error = EAGAIN;
goto out;
}
newval = (lgenval + PTHRW_INC);
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
return(0);
} else
goto loop;
out:
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
#ifdef NOTYET
int
_new_pthread_rwlock_longrdlock_np(pthread_rwlock_t * orwlock)
{
pthread_t self;
uint32_t lgenval, ugenval, rw_wc, newval, updateval;
int error = 0, ret;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
if (is_rw_ewuybit_clear(lgenval))
goto gotlock;
if (is_rw_ebit_set(lgenval)) {
self = pthread_self();
if(rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
newval = (lgenval + PTHRW_INC);
oldval64 = (((uint64_t)rw_wc) << 32);
oldval64 |= lgenval;
newval64 = (((uint64_t)(rw_wc + 1)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lseqaddr) != TRUE)
goto loop;
kblock:
updateval = __psynch_rw_longrdlock(orwlock, newval, ugenval, (rw_wc+1), rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto kblock;
OSAtomicDecrement32((volatile int32_t *)wcaddr);
if (error == 0) {
if ((updateval & PTHRW_RW_HUNLOCK) != 0) {
ret = rwlock_unlock_action_onread(orwlock, (updateval & ~PTHRW_RW_HUNLOCK));
if (ret != 0) {
LIBC_ABORT("rdlock_unlock handling failed");
}
}
error = FALSE;
while (error == FALSE) {
lgenval = *lseqaddr;
newval = lgenval | PTHRW_LBIT;
error = OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr);
}
goto successout;
} else
goto out;
goto successout;
gotlock:
newval = ((lgenval + PTHRW_INC)| PTHRW_LBIT);
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) != TRUE)
goto loop;
successout:
PLOCKSTAT_RW_ACQUIRE(orwlock, READ_LOCK_PLOCKSTAT);
return(0);
out:
PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, error);
return(error);
}
#endif
int
_new_pthread_rwlock_trywrlock(pthread_rwlock_t * orwlock)
{
int error = 0;
uint32_t lgenval, newval;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
lgenval = PTHRW_RW_INIT;
newval = PTHRW_RW_INIT | PTHRW_INC | PTHRW_EBIT;
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
return(0);
}
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EBUSY);
return(EBUSY);
}
int
_new_pthread_rwlock_wrlock(pthread_rwlock_t * orwlock)
{
uint32_t lgenval, newval, ugenval, updateval, rw_wc;
int error = 0;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, lgenval, ugenval, rw_wc, 0);
#endif
#if __DARWIN_UNIX03
if (is_rw_ebit_set(lgenval)) {
if(rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
if (lgenval == PTHRW_RW_INIT) {
newval = ( PTHRW_RW_INIT | PTHRW_INC | PTHRW_EBIT);
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
goto gotit;
}
}
newval = (lgenval + PTHRW_INC) | PTHRW_WBIT | PTHRW_SHADOW_W;
oldval64 = (((uint64_t)rw_wc) << 32);
oldval64 |= lgenval;
newval64 = (((uint64_t)(rw_wc + 1)) << 32);
newval64 |= newval;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lgenval, newval, 0);
#endif
if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lseqaddr) != TRUE)
goto loop;
retry:
PLOCKSTAT_RW_BLOCK(orwlock, WRITE_LOCK_PLOCKSTAT);
retry1:
updateval = __psynch_rw_wrlock(orwlock, newval, ugenval, (rw_wc+1), rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR) {
goto retry1;
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x33333333, newval, updateval, 0);
#endif
PLOCKSTAT_RW_BLOCKED(orwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
if (error != 0) {
OSAtomicDecrement32((volatile int32_t *)wcaddr);
goto out;
}
if (is_rw_ebit_clear(updateval)) {
abort();
}
error = rwlock_exclusive_lockreturn(orwlock, updateval);
if (error == EAGAIN)
goto retry;
OSAtomicDecrement32((volatile int32_t *)wcaddr);
if (error == 0) {
gotit:
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(0);
}
out:
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_WRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(error);
}
#ifdef NOTYET
int
_new_pthread_rwlock_yieldwrlock_np(pthread_rwlock_t * orwlock)
{
uint32_t lgenval, newval, ugenval, updateval, rw_wc;
int error = 0;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
#if __DARWIN_UNIX03
if (is_rw_ebit_set(lgenval)) {
if (rwlock->rw_owner == self) {
error = EDEADLK;
goto out;
}
}
#endif
if (lgenval == PTHRW_RW_INIT) {
newval = PTHRW_RW_INIT | PTHRW_INC | PTHRW_EBIT;
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
goto gotit;
}
}
newval = (lgenval + PTHRW_INC);
if ((lgenval & PTHRW_WBIT) == 0)
newval |= PTHRW_YBIT;
oldval64 = (((uint64_t)rw_wc) << 32);
oldval64 |= lgenval;
newval64 = (((uint64_t)(rw_wc + 1)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lseqaddr) != TRUE)
PLOCKSTAT_RW_BLOCK(orwlock, WRITE_LOCK_PLOCKSTAT);
retry:
updateval = __psynch_rw_yieldwrlock(orwlock, newval, ugenval, (rw_wc+1), rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
PLOCKSTAT_RW_BLOCKED(orwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
if (error != 0) {
OSAtomicDecrement32((volatile int32_t *)wcaddr);
goto out;
}
if (is_rw_ebit_clear(updateval)) {
abort();
}
error = rwlock_exclusive_lockreturn(orwlock, updateval);
if (error == EAGAIN)
goto retry;
OSAtomicDecrement32((volatile int32_t *)wcaddr);
if (error == 0) {
gotit:
#if __DARWIN_UNIX03
rwlock->rw_owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
return(0);
} else {
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
}
return(error);
out:
PLOCKSTAT_RW_ERROR(orwlock, WRITE_LOCK_PLOCKSTAT, error);
return(error);
}
#endif
int
_new_pthread_rwlock_unlock(pthread_rwlock_t * orwlock)
{
uint32_t lgenval, ugenval, rw_wc, newval, nlval, ulval;
int error = 0;
int wrlock = 0, kern_trans;
uint32_t updateval, bits, newbits;
uint32_t isupgrade = 0;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
int retry_count = 0, retry_count1 = 0;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
pthread_t self = NULL;
uint64_t threadid = 0;
int ubitchanged = 0, initbitset = 0, num;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(orwlock, wrlock, error);
return(error);
}
} else {
PLOCKSTAT_RW_ERROR(orwlock, wrlock, EINVAL);
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_START, (uint32_t)rwlock, 0, 0, 0, 0);
#endif
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
loop1:
if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK)) {
retry_count++;
sched_yield();
if (retry_count < 1024)
goto loop;
error = EINVAL;
goto out;
}
retry_count = 0;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, lgenval, ugenval, 0);
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, rw_wc, 0, 0);
#endif
if (is_rw_ebit_set(lgenval)) {
wrlock = 1;
#if __DARWIN_UNIX03
rwlock->rw_owner = (pthread_t)0;
#endif
}
if((lgenval & PTHRW_COUNT_MASK) == (ugenval + PTHRW_INC)) {
if (OSAtomicCompareAndSwap32(ugenval, 0, (volatile int32_t *)useqaddr) != TRUE) {
goto loop;
}
if (OSAtomicCompareAndSwap32(lgenval, PTHRW_RW_INIT, (volatile int32_t *)lseqaddr) != TRUE) {
if (OSAtomicCompareAndSwap32(0, ugenval, (volatile int32_t *)useqaddr) != TRUE) {
lp1:
ulval = *useqaddr;
nlval = ugenval+ulval;
if (OSAtomicCompareAndSwap32(ulval, nlval, (volatile int32_t *)useqaddr) != TRUE)
goto lp1;
}
goto loop;
}
goto succout;
}
lp11:
nlval = lgenval & PTHRW_COUNT_MASK;
if (ubitchanged == 0)
ulval = (ugenval + PTHRW_INC) & PTHRW_COUNT_MASK;
else
ulval = ugenval & PTHRW_COUNT_MASK;
num = rw_diffgenseq(nlval, ulval);
kern_trans = ( num == (rw_wc << PTHRW_COUNT_SHIFT));
if ((ubitchanged ==0) && (kern_trans == 0) && (num < (rw_wc << PTHRW_COUNT_SHIFT))) {
retry_count1++;
sched_yield();
if (retry_count1 < 1024)
goto loop;
}
retry_count1 = 0;
if (ubitchanged == 0) {
if (OSAtomicCompareAndSwap32(ugenval, ugenval+PTHRW_INC, (volatile int32_t *)useqaddr) != TRUE)
goto loop;
ubitchanged = 1;
}
if (kern_trans == 0) {
goto succout;
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 1, ugenval+PTHRW_INC, 0);
#endif
initbitset = 0;
bits = lgenval & PTHRW_BIT_MASK;
newbits = bits;
if ((bits & PTHRW_RW_INIT) != 0) {
newbits &= ~PTHRW_RW_INIT;
initbitset = PTHRW_RW_INIT;
}
if (((bits & PTHRW_EBIT) != 0) && ((bits & PTHRW_WBIT) == 0)) {
newbits &= ~PTHRW_EBIT;
}
if ((bits & PTHRW_WBIT) != 0) {
newbits &= ~PTHRW_SHADOW_W;
}
if (bits & PTHRW_LBIT)
newbits &= ~PTHRW_LBIT;
if (bits & PTHRW_UBIT) {
newbits &= ~PTHRW_LBIT;
newbits |= PTHRW_EBIT;
isupgrade = PTHRW_UBIT;
}
newval = (lgenval & PTHRW_COUNT_MASK) | newbits;
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) != TRUE) {
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
goto lp11;
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 2, newval, 0);
#endif
newval |= (isupgrade | initbitset);
updateval = __psynch_rw_unlock(orwlock, newval, ugenval+PTHRW_INC, rw_wc, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if(error != 0) {
if(error != EINTR)
goto out;
}
if ((updateval & (PTHRW_RW_SPURIOUS | PTHRW_EBIT)) != 0) {
goto succout;
}
lp2:
lgenval = *lseqaddr;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 3, lgenval, 0);
#endif
if ((lgenval & PTHRW_COUNT_MASK) == (updateval & PTHRW_COUNT_MASK)) {
if (OSAtomicCompareAndSwap32(lgenval, updateval, (volatile int32_t *)lseqaddr) != TRUE)
goto lp2;
goto succout;
}
if ((lgenval & PTHRW_BIT_MASK) == (updateval & PTHRW_BIT_MASK)) {
goto succout;
}
newval = ((lgenval & PTHRW_UN_BIT_MASK) << PTHRW_COUNT_SHIFT) | (updateval & PTHRW_BIT_MASK);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 4, newval, 0);
#endif
switch (newval) {
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_LBIT << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_EBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_LBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_LBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_LBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
default:
self = pthread_self();
threadid = self->thread_id;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 6, lgenval, 0);
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 7, updateval, 0);
#endif
LIBC_ABORT("incorect state on return 0x%x: lgenval 0x%x, updateval 0x%x; threadid (0x%x)\n", newval, lgenval, updateval, (uint32_t)threadid);
};
if (error != 0)
goto lp2;
succout:
PLOCKSTAT_RW_RELEASE(orwlock, wrlock);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(0);
out:
PLOCKSTAT_RW_ERROR(orwlock, wrlock, error);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0xAAAAAAAA, error, 0, 0);
#endif
return(error);
}
#ifdef NOTYET
int
_new_pthread_rwlock_downgrade_np(pthread_rwlock_t * orwlock)
{
uint32_t lgenval, newval, ugenval, rw_wc;
int error = 0;
pthread_t self = pthread_self();
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
return(error);
}
} else {
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
if ((is_rw_ebit_set(lgenval )) && (rwlock->rw_owner != self)) {
return(EINVAL);
}
if ((lgenval & PTHRW_COUNT_MASK) != ugenval) {
newval = lgenval & ~PTHRW_EBIT;
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
#if __DARWIN_UNIX03
rwlock->rw_owner = 0;
#endif
if (rw_wc != 0) {
error = __psynch_rw_downgrade(orwlock, newval, ugenval, rw_wc, rwlock->rw_flags);
}
return(0);
} else {
goto loop;
}
}
return(EINVAL);
}
int
_new_pthread_rwlock_upgrade_np(pthread_rwlock_t * orwlock)
{
uint32_t lgenval, newval, ugenval, ulval, updateval, rw_wc;
int error = 0, kern_trans;
pthread_t self = pthread_self();
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint64_t oldval64, newval64;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
return(error);
}
} else {
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
if (is_rw_uebit_set(lgenval)) {
return(EINVAL);
}
if ((lgenval & PTHRW_COUNT_MASK) == ugenval)
return(EINVAL);
if (lgenval > ugenval)
ulval = (lgenval & PTHRW_COUNT_MASK) - (ugenval & PTHRW_COUNT_MASK);
else
ulval = (ugenval & PTHRW_COUNT_MASK) - (lgenval & PTHRW_COUNT_MASK);
newval = lgenval | PTHRW_UBIT;
kern_trans = 1;
if (rw_wc != 0) {
if (ulval == ((rw_wc - 1) << PTHRW_COUNT_SHIFT))
kern_trans = 0;
} else if (ulval == 1)
kern_trans = 0;
if (kern_trans == 0) {
newval = ((lgenval | PTHRW_EBIT) & ~PTHRW_LBIT);
} else {
newval = lgenval | PTHRW_UBIT;
}
if (kern_trans == 0) {
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) != TRUE)
goto loop;
} else {
newval = (lgenval + PTHRW_INC);
oldval64 = (((uint64_t)rw_wc) << 32);
oldval64 |= lgenval;
newval64 = (((uint64_t)(rw_wc + 1)) << 32);
newval64 |= newval;
if (OSAtomicCompareAndSwap64(oldval64, newval64, (volatile int64_t *)lseqaddr) != TRUE)
goto loop;
retry:
updateval = __psynch_rw_upgrade(orwlock, newval, ugenval, rw_wc+1, rwlock->rw_flags);
if (updateval == (uint32_t)-1) {
error = errno;
} else
error = 0;
if (error == EINTR)
goto retry;
if (error != 0) {
OSAtomicDecrement32((volatile int32_t *)wcaddr);
goto out;
}
if (is_rw_ebit_set(updateval)) {
abort();
}
error = rwlock_exclusive_lockreturn(orwlock, updateval);
if (error == EAGAIN)
goto retry;
OSAtomicDecrement32((volatile int32_t *)wcaddr);
}
if (error == 0) {
rwlock->rw_owner = self;
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
return(0);
}
out:
return(error);
}
int
pthread_rwlock_tryupgrade_np(pthread_rwlock_t *orwlock)
{
pthread_t self = pthread_self();
uint32_t lgenval, newval, ugenval, ulval, rw_wc;
int error = 0, kern_trans;
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
return(error);
}
} else {
return(EINVAL);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
loop:
lgenval = *lseqaddr;
ugenval = *useqaddr;
rw_wc = *wcaddr;
if (is_rw_uebit_set(lgenval)) {
return(EBUSY);
}
if ((lgenval & PTHRW_COUNT_MASK) == ugenval)
return(EINVAL);
if (lgenval > ugenval)
ulval = (lgenval & PTHRW_COUNT_MASK) - (ugenval & PTHRW_COUNT_MASK);
else
ulval = (ugenval & PTHRW_COUNT_MASK) - (lgenval & PTHRW_COUNT_MASK);
newval = lgenval | PTHRW_UBIT;
kern_trans = 1;
if (rw_wc != 0) {
if (ulval == (rw_wc - 1))
kern_trans = 0;
} else if (ulval == 1)
kern_trans = 0;
if (kern_trans == 0) {
newval = (lgenval | PTHRW_EBIT) & ~PTHRW_LBIT;
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) != TRUE)
goto loop;
rwlock->rw_owner = self;
PLOCKSTAT_RW_ACQUIRE(orwlock, WRITE_LOCK_PLOCKSTAT);
return(0);
}
return(EBUSY);
}
int
pthread_rwlock_held_np(pthread_rwlock_t * orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lgenval, ugenval;
int error = 0;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
return(0);
}
} else {
return(-1);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
lgenval = *lseqaddr;
ugenval = *useqaddr;
if ((lgenval & PTHRW_COUNT_MASK) == (ugenval & PTHRW_COUNT_MASK))
return(0);
return(1);
}
int
pthread_rwlock_rdheld_np(pthread_rwlock_t * orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lgenval;
int error = 0;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
return(0);
}
} else {
return(-1);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
lgenval = *lseqaddr;
if (is_rw_ebit_set(lgenval)) {
return(0);
}
return(0);
}
int
pthread_rwlock_wrheld_np(pthread_rwlock_t * orwlock)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
pthread_t self;
uint32_t lgenval;
int error = 0;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((error = pthread_rwlock_init(orwlock, NULL)) != 0) {
return(0);
}
} else {
return(-1);
}
}
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
self = pthread_self();
lgenval = *lseqaddr;
if ((is_rw_ebit_set(lgenval)) && (rwlock->rw_owner == self)) {
return(1);
}
return(0);
}
#endif
static int
rwlock_unlock_action_onread(pthread_rwlock_t * orwlock, uint32_t updateval)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
int error = 0;
uint32_t lgenval, newval;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
pthread_t self;
uint64_t threadid;
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
lgenval = *lseqaddr;
lp2:
lgenval = *lseqaddr;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 3, lgenval, 0);
#endif
if ((lgenval & PTHRW_COUNT_MASK) == (updateval & PTHRW_COUNT_MASK)) {
if (OSAtomicCompareAndSwap32(lgenval, updateval, (volatile int32_t *)lseqaddr) != TRUE)
goto lp2;
goto succout;
}
if ((lgenval & PTHRW_BIT_MASK) == (updateval & PTHRW_BIT_MASK)) {
goto succout;
}
newval = ((lgenval & PTHRW_UN_BIT_MASK) << PTHRW_COUNT_SHIFT) | (updateval & PTHRW_BIT_MASK);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 4, newval, 0);
#endif
switch (newval) {
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_LBIT << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action_k(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_EBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action2(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_LBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_LBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_LBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_LBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
default:
self = pthread_self();
threadid = self->thread_id;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 6, lgenval, 0);
(void)__kdebug_trace(_KSYN_TRACE_RW_UNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 7, updateval, 0);
#endif
LIBC_ABORT("incorect state on return 0x%x: lgenval 0x%x, updateval 0x%x; threadid (0x%x)\n", newval, lgenval, updateval, (uint32_t)threadid);
};
if (error != 0)
goto lp2;
succout:
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_NONE, lgenval, newval, 0, 0, 0);
#endif
return(0);
}
static uint32_t
modbits(uint32_t lgenval, uint32_t updateval)
{
uint32_t lval = lgenval & PTHRW_BIT_MASK;
uint32_t uval = updateval & PTHRW_BIT_MASK;
uint32_t rval, nlval;
nlval = (lval | uval);
if ((uval & PTHRW_EBIT) == 0)
nlval &= ~PTHRW_EBIT;
if ((nlval & (PTHRW_WBIT | PTHRW_YBIT)) == (PTHRW_WBIT | PTHRW_YBIT))
nlval &= ~PTHRW_YBIT;
if (((nlval & (PTHRW_WBIT | PTHRW_SHADOW_W)) == PTHRW_WBIT) && ((updateval & PTHRW_WBIT) == 0))
nlval &= ~PTHRW_WBIT;
rval = (lgenval & PTHRW_COUNT_MASK) | nlval;
return(rval);
}
static int
rwlock_unlock_action1(pthread_rwlock_t * orwlock, uint32_t lgenval, uint32_t updateval)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
int error = 0;
uint32_t newval;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
newval = modbits(lgenval, updateval);
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) != TRUE)
error = EINVAL;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT1 | DBG_FUNC_NONE, lgenval, newval, 0, 0, 0);
#endif
return(error);
}
static int
rwlock_unlock_action2(pthread_rwlock_t * orwlock, uint32_t lgenval, uint32_t updateval)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t newval;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
newval = modbits(lgenval, updateval);
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT2 | DBG_FUNC_NONE, lgenval, newval, 0, 0, 0);
#endif
(void) __psynch_rw_unlock2(orwlock, lgenval, *useqaddr, *wcaddr, rwlock->rw_flags);
return(0);
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACT2 | DBG_FUNC_NONE, 0xffffffff, 0, 0, 0, 0);
#endif
return(EINVAL);
}
static int
rwlock_unlock_action_k(pthread_rwlock_t * orwlock, uint32_t lgenval, uint32_t updateval)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t newval;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
newval = modbits(lgenval, updateval);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTK | DBG_FUNC_NONE, lgenval, updateval, newval, 0, 0);
#endif
if (OSAtomicCompareAndSwap32(lgenval, newval, (volatile int32_t *)lseqaddr) == TRUE) {
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTK | DBG_FUNC_NONE, 0x55555555, lgenval, newval, 0, 0);
#endif
} else {
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTK | DBG_FUNC_NONE, 0xAAAAAAAA, lgenval, newval, 0, 0);
#endif
}
return(0);
}
static int
rwlock_exclusive_lockreturn(pthread_rwlock_t * orwlock, uint32_t updateval)
{
npthread_rwlock_t * rwlock = (npthread_rwlock_t *)orwlock;
uint32_t lgenval, newval;
volatile uint32_t * lseqaddr, *useqaddr, *wcaddr;
pthread_t self;
uint64_t threadid;
int error = 0;
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
RWLOCK_GETSEQ_ADDR(rwlock, lseqaddr, useqaddr, wcaddr);
} else {
lseqaddr = rwlock->rw_lseqaddr;
useqaddr = rwlock->rw_useqaddr;
wcaddr = rwlock->rw_wcaddr;
}
lp2:
lgenval = *lseqaddr;
if ((lgenval & PTHRW_COUNT_MASK) == (updateval & PTHRW_COUNT_MASK)) {
if (OSAtomicCompareAndSwap32(lgenval, updateval, (volatile int32_t *)lseqaddr) != TRUE)
goto lp2;
goto out;
}
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTE | DBG_FUNC_NONE, lgenval, updateval, 1, 0, 0);
#endif
if ((lgenval & PTHRW_BIT_MASK) == (updateval & PTHRW_BIT_MASK)) {
goto out;
}
newval = ((lgenval & PTHRW_UN_BIT_MASK) << PTHRW_COUNT_SHIFT) | (updateval & PTHRW_BIT_MASK);
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTE | DBG_FUNC_NONE, newval, 0, 2, 0, 0);
#endif
switch (newval) {
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case ((PTHRW_WBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = EAGAIN;
}
break;
case (PTHRW_YBIT << PTHRW_COUNT_SHIFT) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = EAGAIN;
}
break;
case ((PTHRW_YBIT << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = EAGAIN;
}
break;
case ((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = EAGAIN;
}
break;
case (((PTHRW_YBIT | PTHRW_UBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = EAGAIN;
}
break;
case ((PTHRW_EBIT << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_WBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
case (((PTHRW_WBIT | PTHRW_EBIT) << PTHRW_COUNT_SHIFT) | (PTHRW_YBIT | PTHRW_EBIT)) : {
error = rwlock_unlock_action1(orwlock, lgenval, updateval);
}
break;
default:
self = pthread_self();
threadid = self->thread_id;
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTE | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 6, lgenval, 0);
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTE | DBG_FUNC_NONE, (uint32_t)rwlock, 0x55555555, 7, updateval, 0);
#endif
LIBC_ABORT("rwlock_exclusive_lockreturn: incorect state on return 0x%x: lgenval 0x%x, updateval 0x%x; threadid (0x%x)\n", newval, lgenval, updateval, (uint32_t)threadid);
};
if (error == EINVAL)
goto lp2;
out:
#if _KSYN_TRACE_
(void)__kdebug_trace(_KSYN_TRACE_RW_UNACTE | DBG_FUNC_NONE, error, 0, 0xffffffff, 0, 0);
#endif
return(error);
}
static int
rw_diffgenseq(uint32_t x, uint32_t y)
{
uint32_t lx = (x & PTHRW_COUNT_MASK);
uint32_t ly = (y &PTHRW_COUNT_MASK);
if (lx > ly) {
return(lx-ly);
} else {
return((PTHRW_MAX_READERS - y) + lx + PTHRW_INC);
}
}
#endif
#endif
int
pthread_rwlock_destroy(pthread_rwlock_t *rwlock)
{
#if defined(__i386__) || defined(__x86_64__) || defined(__DARWIN_UNIX03)
int ret;
#endif
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_destroy(rwlock));
}
#endif
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
return(EINVAL);
}
#if defined(__i386__) || defined(__x86_64__)
else if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
ret = _new_pthread_rwlock_destroy(rwlock);
return(ret);
}
#endif
else {
#if __DARWIN_UNIX03
if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0)
return(ret);
if (rwlock->state != 0) {
pthread_mutex_unlock(&rwlock->lock);
return(EBUSY);
}
pthread_mutex_unlock(&rwlock->lock);
#endif
pthread_mutex_destroy(&rwlock->lock);
pthread_cond_destroy(&rwlock->read_signal);
pthread_cond_destroy(&rwlock->write_signal);
rwlock->sig = _PTHREAD_NO_SIG;
return(0);
}
}
int
pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
{
int ret;
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_init(rwlock, attr));
}
#endif
#if __DARWIN_UNIX03
if (attr && (attr->sig != _PTHREAD_RWLOCK_ATTR_SIG)) {
return(EINVAL);
}
#endif
#if defined(__i386__) || defined(__x86_64__)
if ((attr != NULL) && (attr->pshared == PTHREAD_PROCESS_SHARED)) {
ret = _new_pthread_rwlock_init(rwlock, attr);
return(ret);
}
#endif
#if __DARWIN_UNIX03
if ((rwlock->sig == _PTHREAD_RWLOCK_SIG) && (rwlock->state !=0 )) {
return(EBUSY);
}
#endif
if ((ret = pthread_mutex_init(&rwlock->lock, NULL)) != 0)
return(ret);
else {
ret = pthread_cond_init(&rwlock->read_signal, NULL);
if (ret != 0) {
pthread_mutex_destroy(&rwlock->lock);
return(ret);
} else {
ret = pthread_cond_init(&rwlock->write_signal, NULL);
if (ret != 0) {
pthread_cond_destroy(&rwlock->read_signal);
pthread_mutex_destroy(&rwlock->lock);
return(ret);
} else {
rwlock->state = 0;
rwlock->owner = (pthread_t)0;
rwlock->blocked_writers = 0;
if (attr)
rwlock->pshared = attr->pshared;
else
rwlock->pshared = _PTHREAD_DEFAULT_PSHARED;
rwlock->sig = _PTHREAD_RWLOCK_SIG;
return(0);
}
}
}
}
int
pthread_rwlock_rdlock(pthread_rwlock_t *rwlock)
{
int ret;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_rdlock(rwlock));
}
#endif
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
return(ret);
}
}
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
#if defined(__i386__) || defined(__x86_64__)
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
ret = _new_pthread_rwlock_rdlock(rwlock);
return(ret);
}
#endif
if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
return(ret);
}
#if __DARWIN_UNIX03
if ((rwlock->state < 0) && (rwlock->owner == self)) {
pthread_mutex_unlock(&rwlock->lock);
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, EDEADLK);
return(EDEADLK);
}
#endif
#if __DARWIN_UNIX03
while (rwlock->blocked_writers || ((rwlock->state < 0) && (rwlock->owner != self)))
#else
while (rwlock->blocked_writers || rwlock->state < 0)
#endif
{
PLOCKSTAT_RW_BLOCK(rwlock, READ_LOCK_PLOCKSTAT);
ret = pthread_cond_wait(&rwlock->read_signal, &rwlock->lock);
if (ret != 0) {
pthread_mutex_unlock(&rwlock->lock);
PLOCKSTAT_RW_BLOCKED(rwlock, READ_LOCK_PLOCKSTAT, BLOCK_FAIL_PLOCKSTAT);
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
return(ret);
}
PLOCKSTAT_RW_BLOCKED(rwlock, READ_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
}
if (rwlock->state == MAX_READ_LOCKS) {
ret = EAGAIN;
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
}
else {
++rwlock->state;
PLOCKSTAT_RW_ACQUIRE(rwlock, READ_LOCK_PLOCKSTAT);
}
pthread_mutex_unlock(&rwlock->lock);
return(ret);
}
int
pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
{
int ret;
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_tryrdlock(rwlock));
}
#endif
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
return(ret);
}
}
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
#if defined(__i386__) || defined(__x86_64__)
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
ret = _new_pthread_rwlock_tryrdlock(rwlock);
return(ret);
}
#endif
if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
return(ret);
}
if (rwlock->blocked_writers || rwlock->state < 0) {
ret = EBUSY;
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
}
else if (rwlock->state == MAX_READ_LOCKS) {
ret = EAGAIN;
PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, ret);
}
else {
++rwlock->state;
PLOCKSTAT_RW_ACQUIRE(rwlock, READ_LOCK_PLOCKSTAT);
}
pthread_mutex_unlock(&rwlock->lock);
return(ret);
}
int
pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
{
int ret;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_trywrlock(rwlock));
}
#endif
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, ret);
return(ret);
}
}
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
#if defined(__i386__) || defined(__x86_64__)
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
ret = _new_pthread_rwlock_trywrlock(rwlock);
return(ret);
}
#endif
if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, ret);
return(ret);
}
if (rwlock->state != 0) {
ret = EBUSY;
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, ret);
}
else {
rwlock->state = -1;
#if __DARWIN_UNIX03
rwlock->owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(rwlock, WRITE_LOCK_PLOCKSTAT);
}
pthread_mutex_unlock(&rwlock->lock);
return(ret);
}
int
pthread_rwlock_unlock(pthread_rwlock_t *rwlock)
{
int ret;
int writer = (rwlock < 0) ? 1:0;
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_unlock(rwlock));
}
#endif
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
PLOCKSTAT_RW_ERROR(rwlock, writer, EINVAL);
return(EINVAL);
}
#if defined(__i386__) || defined(__x86_64__)
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
ret = _new_pthread_rwlock_unlock(rwlock);
return(ret);
}
#endif
if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, writer, ret);
return(ret);
}
if (rwlock->state > 0) {
if (--rwlock->state == 0 && rwlock->blocked_writers)
ret = pthread_cond_signal(&rwlock->write_signal);
} else if (rwlock->state < 0) {
rwlock->state = 0;
#if __DARWIN_UNIX03
rwlock->owner = (pthread_t)0;
#endif
if (rwlock->blocked_writers)
ret = pthread_cond_signal(&rwlock->write_signal);
else
ret = pthread_cond_broadcast(&rwlock->read_signal);
} else
ret = EINVAL;
if (ret == 0) {
PLOCKSTAT_RW_RELEASE(rwlock, writer);
} else {
PLOCKSTAT_RW_ERROR(rwlock, writer, ret);
}
pthread_mutex_unlock(&rwlock->lock);
return(ret);
}
int
pthread_rwlock_wrlock(pthread_rwlock_t *rwlock)
{
int ret;
#if __DARWIN_UNIX03
pthread_t self = pthread_self();
#endif
#if defined(__i386__) || defined(__x86_64__)
if ((usenew_impl != 0)) {
return(_new_pthread_rwlock_wrlock(rwlock));
}
#endif
if (rwlock->sig == _PTHREAD_RWLOCK_SIG_init) {
if ((ret = pthread_rwlock_init(rwlock, NULL)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, ret);
return(ret);
}
}
if (rwlock->sig != _PTHREAD_RWLOCK_SIG) {
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, EINVAL);
return(EINVAL);
}
#if defined(__i386__) || defined(__x86_64__)
if (rwlock->pshared == PTHREAD_PROCESS_SHARED) {
ret = _new_pthread_rwlock_wrlock(rwlock);
return(ret);
}
#endif
if ((ret = pthread_mutex_lock(&rwlock->lock)) != 0) {
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, ret);
return(ret);
}
#if __DARWIN_UNIX03
if ((rwlock->state < 0) && (rwlock->owner == self)) {
pthread_mutex_unlock(&rwlock->lock);
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, EDEADLK);
return(EDEADLK);
}
#endif
while (rwlock->state != 0) {
++rwlock->blocked_writers;
PLOCKSTAT_RW_BLOCK(rwlock, WRITE_LOCK_PLOCKSTAT);
ret = pthread_cond_wait(&rwlock->write_signal, &rwlock->lock);
if (ret != 0) {
--rwlock->blocked_writers;
pthread_mutex_unlock(&rwlock->lock);
PLOCKSTAT_RW_BLOCKED(rwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_FAIL_PLOCKSTAT);
PLOCKSTAT_RW_ERROR(rwlock, WRITE_LOCK_PLOCKSTAT, ret);
return(ret);
}
PLOCKSTAT_RW_BLOCKED(rwlock, WRITE_LOCK_PLOCKSTAT, BLOCK_SUCCESS_PLOCKSTAT);
--rwlock->blocked_writers;
}
rwlock->state = -1;
#if __DARWIN_UNIX03
rwlock->owner = self;
#endif
PLOCKSTAT_RW_ACQUIRE(rwlock, WRITE_LOCK_PLOCKSTAT);
pthread_mutex_unlock(&rwlock->lock);
return(ret);
}