OSAtomicOperations.c [plain text]
#include <libkern/OSAtomic.h>
#include <kern/debug.h>
#include <machine/atomic.h>
enum {
false = 0,
true = 1
};
#ifndef NULL
#define NULL ((void *)0)
#endif
#define ATOMIC_DEBUG DEBUG
#if ATOMIC_DEBUG
#define ALIGN_TEST(p,t) do{if((uintptr_t)p&(sizeof(t)-1)) panic("Unaligned atomic pointer %p\n",p);}while(0)
#else
#define ALIGN_TEST(p,t) do{}while(0)
#endif
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wcast-qual"
#undef OSCompareAndSwap8
Boolean OSCompareAndSwap8(UInt8 oldValue, UInt8 newValue, volatile UInt8 *address)
{
return __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)address, &oldValue, newValue,
memory_order_acq_rel_smp, memory_order_relaxed);
}
#undef OSCompareAndSwap16
Boolean OSCompareAndSwap16(UInt16 oldValue, UInt16 newValue, volatile UInt16 *address)
{
return __c11_atomic_compare_exchange_strong((_Atomic UInt16 *)address, &oldValue, newValue,
memory_order_acq_rel_smp, memory_order_relaxed);
}
#undef OSCompareAndSwap
Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue, volatile UInt32 *address)
{
ALIGN_TEST(address, UInt32);
return __c11_atomic_compare_exchange_strong((_Atomic UInt32 *)address, &oldValue, newValue,
memory_order_acq_rel_smp, memory_order_relaxed);
}
#undef OSCompareAndSwap64
Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue, volatile UInt64 *address)
{
_Atomic UInt64 *aligned_addr = (_Atomic UInt64 *)(uintptr_t)address;
ALIGN_TEST(address, UInt64);
return __c11_atomic_compare_exchange_strong(aligned_addr, &oldValue, newValue,
memory_order_acq_rel_smp, memory_order_relaxed);
}
#undef OSCompareAndSwapPtr
Boolean OSCompareAndSwapPtr(void *oldValue, void *newValue, void * volatile *address)
{
#if __LP64__
return OSCompareAndSwap64((UInt64)oldValue, (UInt64)newValue, (volatile UInt64 *)address);
#else
return OSCompareAndSwap((UInt32)oldValue, (UInt32)newValue, (volatile UInt32 *)address);
#endif
}
SInt8 OSAddAtomic8(SInt32 amount, volatile SInt8 *address)
{
return __c11_atomic_fetch_add((_Atomic SInt8*)address, amount, memory_order_relaxed);
}
SInt16 OSAddAtomic16(SInt32 amount, volatile SInt16 *address)
{
return __c11_atomic_fetch_add((_Atomic SInt16*)address, amount, memory_order_relaxed);
}
#undef OSAddAtomic
SInt32 OSAddAtomic(SInt32 amount, volatile SInt32 *address)
{
ALIGN_TEST(address, UInt32);
return __c11_atomic_fetch_add((_Atomic SInt32*)address, amount, memory_order_relaxed);
}
#undef OSAddAtomic64
SInt64 OSAddAtomic64(SInt64 amount, volatile SInt64 *address)
{
_Atomic SInt64* aligned_address = (_Atomic SInt64*)(uintptr_t)address;
ALIGN_TEST(address, SInt64);
return __c11_atomic_fetch_add(aligned_address, amount, memory_order_relaxed);
}
#undef OSAddAtomicLong
long
OSAddAtomicLong(long theAmount, volatile long *address)
{
#ifdef __LP64__
return (long)OSAddAtomic64((SInt64)theAmount, (SInt64*)address);
#else
return (long)OSAddAtomic((SInt32)theAmount, address);
#endif
}
#undef OSIncrementAtomic
SInt32 OSIncrementAtomic(volatile SInt32 * value)
{
return OSAddAtomic(1, value);
}
#undef OSDecrementAtomic
SInt32 OSDecrementAtomic(volatile SInt32 * value)
{
return OSAddAtomic(-1, value);
}
#undef OSBitAndAtomic
UInt32 OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
{
return __c11_atomic_fetch_and((_Atomic UInt32*)value, mask, memory_order_relaxed);
}
#undef OSBitOrAtomic
UInt32 OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
{
return __c11_atomic_fetch_or((_Atomic UInt32*)value, mask, memory_order_relaxed);
}
#undef OSBitXorAtomic
UInt32 OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
{
return __c11_atomic_fetch_xor((_Atomic UInt32*)value, mask, memory_order_relaxed);
}
static Boolean OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
{
UInt8 mask = 1;
UInt8 oldValue;
UInt8 wantValue;
startAddress += (bit / 8);
mask <<= (7 - (bit % 8));
wantValue = wantSet ? mask : 0;
do {
oldValue = *startAddress;
if ((oldValue & mask) == wantValue) {
break;
}
} while (! __c11_atomic_compare_exchange_strong((_Atomic UInt8 *)startAddress,
&oldValue, (oldValue & ~mask) | wantValue, memory_order_relaxed, memory_order_relaxed));
return (oldValue & mask) == wantValue;
}
Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
{
return OSTestAndSetClear(bit, true, startAddress);
}
Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
{
return OSTestAndSetClear(bit, false, startAddress);
}
SInt8 OSIncrementAtomic8(volatile SInt8 * value)
{
return OSAddAtomic8(1, value);
}
SInt8 OSDecrementAtomic8(volatile SInt8 * value)
{
return OSAddAtomic8(-1, value);
}
UInt8 OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
{
return __c11_atomic_fetch_and((_Atomic UInt8 *)value, mask, memory_order_relaxed);
}
UInt8 OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
{
return __c11_atomic_fetch_or((_Atomic UInt8 *)value, mask, memory_order_relaxed);
}
UInt8 OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
{
return __c11_atomic_fetch_xor((_Atomic UInt8 *)value, mask, memory_order_relaxed);
}
SInt16 OSIncrementAtomic16(volatile SInt16 * value)
{
return OSAddAtomic16(1, value);
}
SInt16 OSDecrementAtomic16(volatile SInt16 * value)
{
return OSAddAtomic16(-1, value);
}
UInt16 OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
{
return __c11_atomic_fetch_and((_Atomic UInt16 *)value, mask, memory_order_relaxed);
}
UInt16 OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
{
return __c11_atomic_fetch_or((_Atomic UInt16 *)value, mask, memory_order_relaxed);
}
UInt16 OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
{
return __c11_atomic_fetch_xor((_Atomic UInt16 *)value, mask, memory_order_relaxed);
}
#pragma clang diagnostic pop