#ifndef _MACHINE_ATOMIC_H
#error "Do not include <machine/atomic_impl.h> directly, use <machine/atomic.h>"
#endif
#ifndef _MACHINE_ATOMIC_IMPL_H
#define _MACHINE_ATOMIC_IMPL_H
#include <stdatomic.h>
#include <machine/smp.h>
static inline int
memory_order_has_acquire(enum memory_order ord)
{
switch (ord) {
case memory_order_consume:
case memory_order_acquire:
case memory_order_acq_rel:
case memory_order_seq_cst:
return 1;
default:
return 0;
}
}
static inline int
memory_order_has_release(enum memory_order ord)
{
switch (ord) {
case memory_order_release:
case memory_order_acq_rel:
case memory_order_seq_cst:
return 1;
default:
return 0;
}
}
#if __SMP__
#define memory_order_relaxed_smp memory_order_relaxed
#define memory_order_compiler_acquire_smp memory_order_relaxed
#define memory_order_compiler_release_smp memory_order_relaxed
#define memory_order_compiler_acq_rel_smp memory_order_relaxed
#define memory_order_consume_smp memory_order_consume
#define memory_order_dependency_smp memory_order_acquire
#define memory_order_acquire_smp memory_order_acquire
#define memory_order_release_smp memory_order_release
#define memory_order_acq_rel_smp memory_order_acq_rel
#define memory_order_seq_cst_smp memory_order_seq_cst
#else
#define memory_order_relaxed_smp memory_order_relaxed
#define memory_order_compiler_acquire_smp memory_order_relaxed
#define memory_order_compiler_release_smp memory_order_relaxed
#define memory_order_compiler_acq_rel_smp memory_order_relaxed
#define memory_order_consume_smp memory_order_relaxed
#define memory_order_dependency_smp memory_order_relaxed
#define memory_order_acquire_smp memory_order_relaxed
#define memory_order_release_smp memory_order_relaxed
#define memory_order_acq_rel_smp memory_order_relaxed
#define memory_order_seq_cst_smp memory_order_relaxed
#endif
#define _os_compiler_barrier_relaxed memory_order_relaxed
#define _os_compiler_barrier_acquire memory_order_acquire
#define _os_compiler_barrier_release memory_order_release
#define _os_compiler_barrier_acq_rel memory_order_acq_rel
#define _os_compiler_barrier_ memory_order_acq_rel
#define _os_rel_barrier_relaxed memory_order_relaxed
#define _os_rel_barrier_compiler_acquire memory_order_relaxed
#define _os_rel_barrier_compiler_release memory_order_release
#define _os_rel_barrier_compiler_acq_rel memory_order_release
#define _os_rel_barrier_consume memory_order_relaxed
#define _os_rel_barrier_dependency memory_order_relaxed
#define _os_rel_barrier_acquire memory_order_relaxed
#define _os_rel_barrier_release memory_order_release
#define _os_rel_barrier_acq_rel memory_order_release
#define _os_rel_barrier_seq_cst memory_order_release
#define _os_acq_barrier_relaxed memory_order_relaxed
#define _os_acq_barrier_compiler_acquire memory_order_acquire
#define _os_acq_barrier_compiler_release memory_order_relaxed
#define _os_acq_barrier_compiler_acq_rel memory_order_acquire
#define _os_acq_barrier_consume memory_order_acquire
#define _os_acq_barrier_dependency memory_order_acquire
#define _os_acq_barrier_acquire memory_order_acquire
#define _os_acq_barrier_release memory_order_relaxed
#define _os_acq_barrier_acq_rel memory_order_acquire
#define _os_acq_barrier_seq_cst memory_order_acquire
#define _os_compiler_barrier_before_atomic(m) \
atomic_signal_fence(_os_rel_barrier_##m)
#define _os_compiler_barrier_after_atomic(m) \
atomic_signal_fence(_os_acq_barrier_##m)
#define _os_rel_fence_relaxed memory_order_relaxed
#define _os_rel_fence_compiler_acquire memory_order_relaxed
#define _os_rel_fence_compiler_release memory_order_release
#define _os_rel_fence_compiler_acq_rel memory_order_release
#define _os_rel_fence_consume memory_order_relaxed_smp
#define _os_rel_fence_dependency memory_order_relaxed_smp
#define _os_rel_fence_acquire memory_order_relaxed_smp
#define _os_rel_fence_release memory_order_release_smp
#define _os_rel_fence_acq_rel memory_order_release_smp
#define _os_rel_fence_seq_cst memory_order_release_smp
#define _os_acq_fence_relaxed memory_order_relaxed
#define _os_acq_fence_compiler_acquire memory_order_relaxed
#define _os_acq_fence_compiler_release memory_order_relaxed
#define _os_acq_fence_compiler_acq_rel memory_order_relaxed
#define _os_acq_fence_consume memory_order_acquire_smp
#define _os_acq_fence_dependency memory_order_dependency_smp
#define _os_acq_fence_acquire memory_order_acquire_smp
#define _os_acq_fence_release memory_order_relaxed_smp
#define _os_acq_fence_acq_rel memory_order_acquire_smp
#define _os_acq_fence_seq_cst memory_order_acquire_smp
#define _os_memory_fence_before_atomic(m) \
atomic_thread_fence(_os_rel_fence_##m)
#define _os_memory_fence_after_atomic(m) \
atomic_thread_fence(_os_acq_fence_##m)
#define _os_atomic_c11_atomic(p) \
((typeof(*(p)) volatile _Atomic *)(p))
#define _os_atomic_basetypeof(p) \
typeof(atomic_load(_os_atomic_c11_atomic(p)))
#define _os_atomic_op_orig(p, v, m, o) ({ \
_os_atomic_basetypeof(p) _r; \
_os_compiler_barrier_before_atomic(m); \
_r = o(_os_atomic_c11_atomic(p), v, memory_order_##m##_smp); \
_os_compiler_barrier_after_atomic(m); \
_r; \
})
#define _os_atomic_c11_op_orig(p, v, m, o) \
_os_atomic_op_orig(p, v, m, atomic_##o##_explicit)
#define _os_atomic_c11_op(p, v, m, o, op) \
({ typeof(v) _v = (v); _os_atomic_c11_op_orig(p, _v, m, o) op _v; })
#define _os_atomic_clang_op_orig(p, v, m, o) \
_os_atomic_op_orig(p, v, m, __atomic_##o)
#define _os_atomic_clang_op(p, v, m, o, op) \
({ typeof(v) _v = (v); _os_atomic_basetypeof(p) _r = \
_os_atomic_clang_op_orig(p, _v, m, o); op(_r, _v); })
#define _os_atomic_auto_dependency(e) \
_Generic(e, \
os_atomic_dependency_t: (e), \
default: os_atomic_make_dependency(e))
#endif