#include <stdint.h>
#include <string.h>
#if __APPLE__
#pragma redefine_extname __atomic_load_c ___atomic_load
#pragma redefine_extname __atomic_store_c ___atomic_store
#pragma redefine_extname __atomic_exchange_c ___atomic_exchange
#pragma redefine_extname __atomic_compare_exchange_c ___atomic_compare_exchange
#else
#pragma redefine_extname __atomic_load_c __atomic_load
#pragma redefine_extname __atomic_store_c __atomic_store
#pragma redefine_extname __atomic_exchange_c __atomic_exchange
#pragma redefine_extname __atomic_compare_exchange_c __atomic_compare_exchange
#endif
#ifndef SPINLOCK_COUNT
#define SPINLOCK_COUNT (1<<10)
#endif
static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1;
#ifdef __FreeBSD__
#include <errno.h>
#include <sys/types.h>
#include <machine/atomic.h>
#include <sys/umtx.h>
typedef struct _usem Lock;
inline static void unlock(Lock *l) {
__c11_atomic_store((_Atomic(uint32_t)*)&l->_count, 1, __ATOMIC_RELEASE);
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
if (l->_has_waiters)
_umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0);
}
inline static void lock(Lock *l) {
uint32_t old = 1;
while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t)*)&l->_count, &old,
0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
_umtx_op(l, UMTX_OP_SEM_WAIT, 0, 0, 0);
old = 1;
}
}
static Lock locks[SPINLOCK_COUNT] = { [0 ... SPINLOCK_COUNT-1] = {0,1,0} };
#elif defined(__APPLE__)
#include <libkern/OSAtomic.h>
typedef OSSpinLock Lock;
inline static void unlock(Lock *l) {
OSSpinLockUnlock(l);
}
inline static void lock(Lock *l) {
OSSpinLockLock(l);
}
static Lock locks[SPINLOCK_COUNT];
#else
typedef _Atomic(uintptr_t) Lock;
inline static void unlock(Lock *l) {
__c11_atomic_store(l, 0, __ATOMIC_RELEASE);
}
inline static void lock(Lock *l) {
uintptr_t old = 0;
while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE,
__ATOMIC_RELAXED))
old = 0;
}
static Lock locks[SPINLOCK_COUNT];
#endif
static inline Lock *lock_for_pointer(void *ptr) {
intptr_t hash = (intptr_t)ptr;
hash >>= 4;
intptr_t low = hash & SPINLOCK_MASK;
hash >>= 16;
hash ^= low;
return locks + (hash & SPINLOCK_MASK);
}
#define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1)
#define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2)
#define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4)
#define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8)
#define IS_LOCK_FREE_16 0
#define LOCK_FREE_CASES() \
do {\
switch (size) {\
case 2:\
if (IS_LOCK_FREE_2) {\
LOCK_FREE_ACTION(uint16_t);\
}\
case 4:\
if (IS_LOCK_FREE_4) {\
LOCK_FREE_ACTION(uint32_t);\
}\
case 8:\
if (IS_LOCK_FREE_8) {\
LOCK_FREE_ACTION(uint64_t);\
}\
case 16:\
if (IS_LOCK_FREE_16) {\
\
}\
}\
} while (0)
void __atomic_load_c(int size, void *src, void *dest, int model) {
#define LOCK_FREE_ACTION(type) \
*((type*)dest) = __c11_atomic_load((_Atomic(type)*)src, model);\
return;
LOCK_FREE_CASES();
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(src);
lock(l);
memcpy(dest, src, size);
unlock(l);
}
void __atomic_store_c(int size, void *dest, void *src, int model) {
#define LOCK_FREE_ACTION(type) \
__c11_atomic_store((_Atomic(type)*)dest, *(type*)dest, model);\
return;
LOCK_FREE_CASES();
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(dest);
lock(l);
memcpy(dest, src, size);
unlock(l);
}
int __atomic_compare_exchange_c(int size, void *ptr, void *expected,
void *desired, int success, int failure) {
#define LOCK_FREE_ACTION(type) \
return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, (type*)expected,\
*(type*)desired, success, failure)
LOCK_FREE_CASES();
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
if (memcmp(ptr, expected, size) == 0) {
memcpy(ptr, desired, size);
unlock(l);
return 1;
}
memcpy(expected, ptr, size);
unlock(l);
return 0;
}
void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) {
#define LOCK_FREE_ACTION(type) \
*(type*)old = __c11_atomic_exchange((_Atomic(type)*)ptr, *(type*)val,\
model);\
return;
LOCK_FREE_CASES();
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
memcpy(old, ptr, size);
memcpy(ptr, val, size);
unlock(l);
}
#define OPTIMISED_CASES\
OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t)\
OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t)\
OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t)\
OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t)\
\
#define OPTIMISED_CASE(n, lockfree, type)\
type __atomic_load_##n(type *src, int model) {\
if (lockfree)\
return __c11_atomic_load((_Atomic(type)*)src, model);\
Lock *l = lock_for_pointer(src);\
lock(l);\
type val = *src;\
unlock(l);\
return val;\
}
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type)\
void __atomic_store_##n(type *dest, type val, int model) {\
if (lockfree) {\
__c11_atomic_store((_Atomic(type)*)dest, val, model);\
return;\
}\
Lock *l = lock_for_pointer(dest);\
lock(l);\
*dest = val;\
unlock(l);\
return;\
}
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type)\
type __atomic_exchange_##n(type *dest, type val, int model) {\
if (lockfree)\
return __c11_atomic_exchange((_Atomic(type)*)dest, val, model);\
Lock *l = lock_for_pointer(dest);\
lock(l);\
type tmp = *dest;\
*dest = val;\
unlock(l);\
return tmp;\
}
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type)\
int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired,\
int success, int failure) {\
if (lockfree)\
return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, expected, desired,\
success, failure);\
Lock *l = lock_for_pointer(ptr);\
lock(l);\
if (*ptr == *expected) {\
*ptr = desired;\
unlock(l);\
return 1;\
}\
*expected = *ptr;\
unlock(l);\
return 0;\
}
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define ATOMIC_RMW(n, lockfree, type, opname, op) \
type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) {\
if (lockfree) \
return __c11_atomic_fetch_##opname((_Atomic(type)*)ptr, val, model);\
Lock *l = lock_for_pointer(ptr);\
lock(l);\
type tmp = *ptr;\
*ptr = tmp op val;\
unlock(l);\
return tmp;\
}
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +)
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, sub, -)
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, and, &)
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, or, |)
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, xor, ^)
OPTIMISED_CASES
#undef OPTIMISED_CASE