#ifndef Atomics_h
#define Atomics_h
#include <atomic>
#include <wtf/StdLibExtras.h>
#if OS(WINDOWS)
#if !COMPILER(GCC)
extern "C" void _ReadWriteBarrier(void);
#pragma intrinsic(_ReadWriteBarrier)
#endif
#include <windows.h>
#endif
namespace WTF {
template<typename T>
struct Atomic {
T load(std::memory_order order = std::memory_order_seq_cst) const { return value.load(order); }
void store(T desired, std::memory_order order = std::memory_order_seq_cst) { value.store(desired, order); }
bool compareExchangeWeak(T expected, T desired, std::memory_order order = std::memory_order_seq_cst)
{
T expectedOrActual = expected;
return value.compare_exchange_weak(expectedOrActual, desired, order);
}
bool compareExchangeStrong(T expected, T desired, std::memory_order order = std::memory_order_seq_cst)
{
T expectedOrActual = expected;
return value.compare_exchange_strong(expectedOrActual, desired, order);
}
std::atomic<T> value;
};
#if OS(WINDOWS)
inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue)
{
return InterlockedCompareExchange(reinterpret_cast<LONG volatile*>(location), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected);
}
inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue)
{
return InterlockedCompareExchangePointer(location, newValue, expected) == expected;
}
#else // OS(WINDOWS) --> not windows
inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue)
{
#if ENABLE(COMPARE_AND_SWAP)
#if CPU(X86) || CPU(X86_64)
unsigned char result;
asm volatile(
"lock; cmpxchgl %3, %2\n\t"
"sete %1"
: "+a"(expected), "=q"(result), "+m"(*location)
: "r"(newValue)
: "memory"
);
#elif CPU(ARM_THUMB2)
unsigned tmp;
unsigned result;
asm volatile(
"movw %1, #1\n\t"
"ldrex %2, %0\n\t"
"cmp %3, %2\n\t"
"bne.n 0f\n\t"
"strex %1, %4, %0\n\t"
"0:"
: "+Q"(*location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
result = !result;
#elif CPU(ARM64) && COMPILER(GCC)
unsigned tmp;
unsigned result;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %w2, [%0]\n\t"
"cmp %w3, %w2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %w4, [%0]\n\t"
"0:"
: "+r"(location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
result = !result;
#elif CPU(ARM64)
unsigned tmp;
unsigned result;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %w2, %0\n\t"
"cmp %w3, %w2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %w4, %0\n\t"
"0:"
: "+m"(*location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
result = !result;
#else
#error "Bad architecture for compare and swap."
#endif
return result;
#else
UNUSED_PARAM(location);
UNUSED_PARAM(expected);
UNUSED_PARAM(newValue);
CRASH();
return false;
#endif
}
inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue)
{
#if ENABLE(COMPARE_AND_SWAP)
#if CPU(X86_64)
bool result;
asm volatile(
"lock; cmpxchgq %3, %2\n\t"
"sete %1"
: "+a"(expected), "=q"(result), "+m"(*location)
: "r"(newValue)
: "memory"
);
return result;
#elif CPU(ARM64) && COMPILER(GCC)
bool result;
void* tmp;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %x2, [%0]\n\t"
"cmp %x3, %x2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %x4, [%0]\n\t"
"0:"
: "+r"(location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
return !result;
#elif CPU(ARM64)
bool result;
void* tmp;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %x2, %0\n\t"
"cmp %x3, %x2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %x4, %0\n\t"
"0:"
: "+m"(*location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
return !result;
#else
return weakCompareAndSwap(bitwise_cast<unsigned*>(location), bitwise_cast<unsigned>(expected), bitwise_cast<unsigned>(newValue));
#endif
#else // ENABLE(COMPARE_AND_SWAP)
UNUSED_PARAM(location);
UNUSED_PARAM(expected);
UNUSED_PARAM(newValue);
CRASH();
return 0;
#endif // ENABLE(COMPARE_AND_SWAP)
}
#endif // OS(WINDOWS) (end of the not-windows case)
inline bool weakCompareAndSwapUIntPtr(volatile uintptr_t* location, uintptr_t expected, uintptr_t newValue)
{
return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue));
}
inline bool weakCompareAndSwapSize(volatile size_t* location, size_t expected, size_t newValue)
{
return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue));
}
inline void compilerFence()
{
#if OS(WINDOWS) && !COMPILER(GCC)
_ReadWriteBarrier();
#else
asm volatile("" ::: "memory");
#endif
}
#if CPU(ARM_THUMB2) || CPU(ARM64)
inline void armV7_dmb()
{
asm volatile("dmb sy" ::: "memory");
}
inline void armV7_dmb_st()
{
asm volatile("dmb st" ::: "memory");
}
inline void loadLoadFence() { armV7_dmb(); }
inline void loadStoreFence() { armV7_dmb(); }
inline void storeLoadFence() { armV7_dmb(); }
inline void storeStoreFence() { armV7_dmb_st(); }
inline void memoryBarrierAfterLock() { armV7_dmb(); }
inline void memoryBarrierBeforeUnlock() { armV7_dmb(); }
#elif CPU(X86) || CPU(X86_64)
inline void x86_mfence()
{
#if OS(WINDOWS)
MemoryBarrier();
#else
asm volatile("mfence" ::: "memory");
#endif
}
inline void loadLoadFence() { compilerFence(); }
inline void loadStoreFence() { compilerFence(); }
inline void storeLoadFence() { x86_mfence(); }
inline void storeStoreFence() { compilerFence(); }
inline void memoryBarrierAfterLock() { compilerFence(); }
inline void memoryBarrierBeforeUnlock() { compilerFence(); }
#else
inline void loadLoadFence() { compilerFence(); }
inline void loadStoreFence() { compilerFence(); }
inline void storeLoadFence() { compilerFence(); }
inline void storeStoreFence() { compilerFence(); }
inline void memoryBarrierAfterLock() { compilerFence(); }
inline void memoryBarrierBeforeUnlock() { compilerFence(); }
#endif
inline bool weakCompareAndSwap(uint8_t* location, uint8_t expected, uint8_t newValue)
{
#if ENABLE(COMPARE_AND_SWAP)
#if !OS(WINDOWS) && (CPU(X86) || CPU(X86_64))
unsigned char result;
asm volatile(
"lock; cmpxchgb %3, %2\n\t"
"sete %1"
: "+a"(expected), "=q"(result), "+m"(*location)
: "q"(newValue)
: "memory"
);
return result;
#elif OS(WINDOWS) && CPU(X86)
bool result = false;
__asm {
mov al, expected
mov edx, location
mov cl, newValue
lock cmpxchg byte ptr[edx], cl
setz result
}
return result;
#else
uintptr_t locationValue = bitwise_cast<uintptr_t>(location);
uintptr_t alignedLocationValue = locationValue & ~(sizeof(unsigned) - 1);
uintptr_t locationOffset = locationValue - alignedLocationValue;
ASSERT(locationOffset < sizeof(unsigned));
unsigned* alignedLocation = bitwise_cast<unsigned*>(alignedLocationValue);
unsigned oldAlignedValue = *const_cast<volatile unsigned*>(alignedLocation);
struct Splicer {
static unsigned splice(unsigned value, uint8_t byte, uintptr_t byteIndex)
{
union {
unsigned word;
uint8_t bytes[sizeof(unsigned)];
} u;
u.word = value;
u.bytes[byteIndex] = byte;
return u.word;
}
};
unsigned expectedAlignedValue = Splicer::splice(oldAlignedValue, expected, locationOffset);
unsigned newAlignedValue = Splicer::splice(oldAlignedValue, newValue, locationOffset);
return weakCompareAndSwap(alignedLocation, expectedAlignedValue, newAlignedValue);
#endif
#else
UNUSED_PARAM(location);
UNUSED_PARAM(expected);
UNUSED_PARAM(newValue);
CRASH();
return false;
#endif
}
}
using WTF::Atomic;
#endif // Atomics_h