#ifndef Atomics_h
#define Atomics_h
#include <wtf/StdLibExtras.h>
#if OS(WINDOWS)
#if !COMPILER(GCC)
extern "C" void _ReadWriteBarrier(void);
#pragma intrinsic(_ReadWriteBarrier)
#endif
#include <windows.h>
#endif
namespace WTF {
#if OS(WINDOWS)
inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue)
{
#if OS(WINCE)
return InterlockedCompareExchange(reinterpret_cast<LONG*>(const_cast<unsigned*>(location)), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected);
#else
return InterlockedCompareExchange(reinterpret_cast<LONG volatile*>(location), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected);
#endif
}
inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue)
{
return InterlockedCompareExchangePointer(location, newValue, expected) == expected;
}
#else // OS(WINDOWS) --> not windows
inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue)
{
#if ENABLE(COMPARE_AND_SWAP)
#if CPU(X86) || CPU(X86_64)
unsigned char result;
asm volatile(
"lock; cmpxchgl %3, %2\n\t"
"sete %1"
: "+a"(expected), "=q"(result), "+m"(*location)
: "r"(newValue)
: "memory"
);
#elif CPU(ARM_THUMB2)
unsigned tmp;
unsigned result;
asm volatile(
"movw %1, #1\n\t"
"ldrex %2, %0\n\t"
"cmp %3, %2\n\t"
"bne.n 0f\n\t"
"strex %1, %4, %0\n\t"
"0:"
: "+Q"(*location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
result = !result;
#elif CPU(ARM64) && COMPILER(GCC)
unsigned tmp;
unsigned result;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %w2, [%0]\n\t"
"cmp %w3, %w2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %w4, [%0]\n\t"
"0:"
: "+r"(location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
result = !result;
#elif CPU(ARM64)
unsigned tmp;
unsigned result;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %w2, %0\n\t"
"cmp %w3, %w2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %w4, %0\n\t"
"0:"
: "+m"(*location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
result = !result;
#else
#error "Bad architecture for compare and swap."
#endif
return result;
#else
UNUSED_PARAM(location);
UNUSED_PARAM(expected);
UNUSED_PARAM(newValue);
CRASH();
return false;
#endif
}
inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue)
{
#if ENABLE(COMPARE_AND_SWAP)
#if CPU(X86_64)
bool result;
asm volatile(
"lock; cmpxchgq %3, %2\n\t"
"sete %1"
: "+a"(expected), "=q"(result), "+m"(*location)
: "r"(newValue)
: "memory"
);
return result;
#elif CPU(ARM64) && COMPILER(GCC)
bool result;
void* tmp;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %x2, [%0]\n\t"
"cmp %x3, %x2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %x4, [%0]\n\t"
"0:"
: "+r"(location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
return !result;
#elif CPU(ARM64)
bool result;
void* tmp;
asm volatile(
"mov %w1, #1\n\t"
"ldxr %x2, %0\n\t"
"cmp %x3, %x2\n\t"
"b.ne 0f\n\t"
"stxr %w1, %x4, %0\n\t"
"0:"
: "+m"(*location), "=&r"(result), "=&r"(tmp)
: "r"(expected), "r"(newValue)
: "memory");
return !result;
#else
return weakCompareAndSwap(bitwise_cast<unsigned*>(location), bitwise_cast<unsigned>(expected), bitwise_cast<unsigned>(newValue));
#endif
#else // ENABLE(COMPARE_AND_SWAP)
UNUSED_PARAM(location);
UNUSED_PARAM(expected);
UNUSED_PARAM(newValue);
CRASH();
return 0;
#endif // ENABLE(COMPARE_AND_SWAP)
}
#endif // OS(WINDOWS) (end of the not-windows case)
inline bool weakCompareAndSwapUIntPtr(volatile uintptr_t* location, uintptr_t expected, uintptr_t newValue)
{
return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue));
}
inline bool weakCompareAndSwapSize(volatile size_t* location, size_t expected, size_t newValue)
{
return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue));
}
inline void compilerFence()
{
#if OS(WINDOWS) && !COMPILER(GCC)
_ReadWriteBarrier();
#else
asm volatile("" ::: "memory");
#endif
}
#if CPU(ARM_THUMB2) || CPU(ARM64)
inline void armV7_dmb()
{
asm volatile("dmb sy" ::: "memory");
}
inline void armV7_dmb_st()
{
asm volatile("dmb st" ::: "memory");
}
inline void loadLoadFence() { armV7_dmb(); }
inline void loadStoreFence() { armV7_dmb(); }
inline void storeLoadFence() { armV7_dmb(); }
inline void storeStoreFence() { armV7_dmb_st(); }
inline void memoryBarrierAfterLock() { armV7_dmb(); }
inline void memoryBarrierBeforeUnlock() { armV7_dmb(); }
#elif CPU(X86) || CPU(X86_64)
inline void x86_mfence()
{
#if OS(WINDOWS)
MemoryBarrier();
#else
asm volatile("mfence" ::: "memory");
#endif
}
inline void loadLoadFence() { compilerFence(); }
inline void loadStoreFence() { compilerFence(); }
inline void storeLoadFence() { x86_mfence(); }
inline void storeStoreFence() { compilerFence(); }
inline void memoryBarrierAfterLock() { compilerFence(); }
inline void memoryBarrierBeforeUnlock() { compilerFence(); }
#else
inline void loadLoadFence() { compilerFence(); }
inline void loadStoreFence() { compilerFence(); }
inline void storeLoadFence() { compilerFence(); }
inline void storeStoreFence() { compilerFence(); }
inline void memoryBarrierAfterLock() { compilerFence(); }
inline void memoryBarrierBeforeUnlock() { compilerFence(); }
#endif
inline bool weakCompareAndSwap(uint8_t* location, uint8_t expected, uint8_t newValue)
{
#if ENABLE(COMPARE_AND_SWAP)
#if !OS(WINDOWS) && (CPU(X86) || CPU(X86_64))
unsigned char result;
asm volatile(
"lock; cmpxchgb %3, %2\n\t"
"sete %1"
: "+a"(expected), "=q"(result), "+m"(*location)
: "q"(newValue)
: "memory"
);
return result;
#elif OS(WINDOWS) && CPU(X86)
bool result = false;
__asm {
mov al, expected
mov edx, location
mov cl, newValue
lock cmpxchg byte ptr[edx], cl
setz result
}
return result;
#else
uintptr_t locationValue = bitwise_cast<uintptr_t>(location);
uintptr_t alignedLocationValue = locationValue & ~(sizeof(unsigned) - 1);
uintptr_t locationOffset = locationValue - alignedLocationValue;
ASSERT(locationOffset < sizeof(unsigned));
unsigned* alignedLocation = bitwise_cast<unsigned*>(alignedLocationValue);
unsigned oldAlignedValue = *const_cast<volatile unsigned*>(alignedLocation);
union {
uint8_t bytes[sizeof(unsigned)];
unsigned word;
} u;
u.word = oldAlignedValue;
if (u.bytes[locationOffset] != expected)
return false;
u.bytes[locationOffset] = newValue;
unsigned newAlignedValue = u.word;
return weakCompareAndSwap(alignedLocation, oldAlignedValue, newAlignedValue);
#endif
#else
UNUSED_PARAM(location);
UNUSED_PARAM(expected);
UNUSED_PARAM(newValue);
CRASH();
return false;
#endif
}
}
#endif // Atomics_h