atomic_private_impl.h [plain text]
#ifndef __OS_ATOMIC_PRIVATE_H__
#error "Do not include <os/atomic_private_impl.h> directly, use <os/atomic_private.h>"
#endif
#ifndef __OS_ATOMIC_PRIVATE_IMPL_H__
#define __OS_ATOMIC_PRIVATE_IMPL_H__
#pragma mark - implementation details
static inline int
_os_atomic_mo_has_acquire(OS_ATOMIC_STD memory_order ord)
{
switch (ord) {
case os_atomic_std(memory_order_consume):
case os_atomic_std(memory_order_acquire):
case os_atomic_std(memory_order_acq_rel):
case os_atomic_std(memory_order_seq_cst):
return 1;
default:
return 0;
}
}
static inline int
_os_atomic_mo_has_release(OS_ATOMIC_STD memory_order ord)
{
switch (ord) {
case os_atomic_std(memory_order_release):
case os_atomic_std(memory_order_acq_rel):
case os_atomic_std(memory_order_seq_cst):
return 1;
default:
return 0;
}
}
#define _os_atomic_mo_relaxed os_atomic_std(memory_order_relaxed)
#define _os_atomic_mo_compiler_acquire os_atomic_std(memory_order_relaxed)
#define _os_atomic_mo_compiler_release os_atomic_std(memory_order_relaxed)
#define _os_atomic_mo_compiler_acq_rel os_atomic_std(memory_order_relaxed)
#define _os_atomic_mo_consume os_atomic_std(memory_order_consume)
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_mo_dependency os_atomic_std(memory_order_acquire)
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_mo_acquire os_atomic_std(memory_order_acquire)
#define _os_atomic_mo_release os_atomic_std(memory_order_release)
#define _os_atomic_mo_acq_rel os_atomic_std(memory_order_acq_rel)
#define _os_atomic_mo_seq_cst os_atomic_std(memory_order_seq_cst)
#if OS_ATOMIC_CONFIG_SMP
#define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_consume_smp _os_atomic_mo_consume
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_mo_dependency_smp _os_atomic_mo_dependency
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_mo_acquire_smp _os_atomic_mo_acquire
#define _os_atomic_mo_release_smp _os_atomic_mo_release
#define _os_atomic_mo_acq_rel_smp _os_atomic_mo_acq_rel
#define _os_atomic_mo_seq_cst_smp _os_atomic_mo_seq_cst
#else
#define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_consume_smp _os_atomic_mo_relaxed
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_mo_dependency_smp _os_atomic_mo_relaxed
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_mo_acquire_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_release_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_acq_rel_smp _os_atomic_mo_relaxed
#define _os_atomic_mo_seq_cst_smp _os_atomic_mo_relaxed
#endif
#if KERNEL_PRIVATE
#define memory_order_relaxed_smp _os_atomic_mo_relaxed_smp
#define memory_order_compiler_acquire_smp _os_atomic_mo_compiler_acquire_smp
#define memory_order_compiler_release_smp _os_atomic_mo_compiler_release_smp
#define memory_order_compiler_acq_rel_smp _os_atomic_mo_compiler_acq_rel_smp
#define memory_order_consume_smp _os_atomic_mo_consume_smp
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define memory_order_dependency _os_atomic_mo_dependency
#define memory_order_dependency_smp _os_atomic_mo_dependency_smp
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define memory_order_acquire_smp _os_atomic_mo_acquire_smp
#define memory_order_release_smp _os_atomic_mo_release_smp
#define memory_order_acq_rel_smp _os_atomic_mo_acq_rel_smp
#define memory_order_seq_cst_smp _os_atomic_mo_seq_cst_smp
#endif
#define _os_compiler_barrier_relaxed _os_atomic_mo_relaxed
#define _os_compiler_barrier_acquire _os_atomic_mo_acquire
#define _os_compiler_barrier_release _os_atomic_mo_release
#define _os_compiler_barrier_acq_rel _os_atomic_mo_acq_rel
#define _os_compiler_barrier_ _os_atomic_mo_acq_rel
#define _os_rel_barrier_relaxed _os_atomic_mo_relaxed
#define _os_rel_barrier_compiler_acquire _os_atomic_mo_relaxed
#define _os_rel_barrier_compiler_release _os_atomic_mo_release
#define _os_rel_barrier_compiler_acq_rel _os_atomic_mo_release
#define _os_rel_barrier_consume _os_atomic_mo_relaxed
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_rel_barrier_dependency _os_atomic_mo_relaxed
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_rel_barrier_acquire _os_atomic_mo_relaxed
#define _os_rel_barrier_release _os_atomic_mo_release
#define _os_rel_barrier_acq_rel _os_atomic_mo_release
#define _os_rel_barrier_seq_cst _os_atomic_mo_release
#define _os_acq_barrier_relaxed _os_atomic_mo_relaxed
#define _os_acq_barrier_compiler_acquire _os_atomic_mo_acquire
#define _os_acq_barrier_compiler_release _os_atomic_mo_relaxed
#define _os_acq_barrier_compiler_acq_rel _os_atomic_mo_acquire
#define _os_acq_barrier_consume _os_atomic_mo_acquire
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_acq_barrier_dependency _os_atomic_mo_acquire
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_acq_barrier_acquire _os_atomic_mo_acquire
#define _os_acq_barrier_release _os_atomic_mo_relaxed
#define _os_acq_barrier_acq_rel _os_atomic_mo_acquire
#define _os_acq_barrier_seq_cst _os_atomic_mo_acquire
#define _os_compiler_barrier_before_atomic(m) \
os_atomic_std(atomic_signal_fence)(_os_rel_barrier_##m)
#define _os_compiler_barrier_after_atomic(m) \
os_atomic_std(atomic_signal_fence)(_os_acq_barrier_##m)
#define _os_rel_fence_relaxed _os_atomic_mo_relaxed
#define _os_rel_fence_compiler_acquire _os_atomic_mo_relaxed
#define _os_rel_fence_compiler_release _os_atomic_mo_release
#define _os_rel_fence_compiler_acq_rel _os_atomic_mo_release
#define _os_rel_fence_consume _os_atomic_mo_relaxed_smp
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_rel_fence_dependency _os_atomic_mo_relaxed_smp
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_rel_fence_acquire _os_atomic_mo_relaxed_smp
#define _os_rel_fence_release _os_atomic_mo_release_smp
#define _os_rel_fence_acq_rel _os_atomic_mo_release_smp
#define _os_rel_fence_seq_cst _os_atomic_mo_release_smp
#define _os_acq_fence_relaxed _os_atomic_mo_relaxed
#define _os_acq_fence_compiler_acquire _os_atomic_mo_relaxed
#define _os_acq_fence_compiler_release _os_atomic_mo_relaxed
#define _os_acq_fence_compiler_acq_rel _os_atomic_mo_relaxed
#define _os_acq_fence_consume _os_atomic_mo_acquire_smp
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_acq_fence_dependency _os_atomic_mo_dependency_smp
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_acq_fence_acquire _os_atomic_mo_acquire_smp
#define _os_acq_fence_release _os_atomic_mo_relaxed_smp
#define _os_acq_fence_acq_rel _os_atomic_mo_acquire_smp
#define _os_acq_fence_seq_cst _os_atomic_mo_acquire_smp
#define _os_memory_fence_before_atomic(m) \
os_atomic_std(atomic_thread_fence)(_os_rel_fence_##m)
#define _os_memory_fence_after_atomic(m) \
os_atomic_std(atomic_thread_fence)(_os_acq_fence_##m)
#define _os_atomic_value_cast(p, v) \
({ typeof(*os_cast_to_nonatomic_pointer(p)) ___v = (v); ___v; })
#define _os_atomic_c11_op_orig(p, v, m, o) ({ \
_os_compiler_barrier_before_atomic(m); \
__auto_type _r = os_atomic_std(atomic_##o##_explicit)(\
os_cast_to_atomic_pointer(p), \
_os_atomic_value_cast(p, v), \
_os_atomic_mo_##m##_smp); \
_os_compiler_barrier_after_atomic(m); \
_r; \
})
#define _os_atomic_c11_op(p, v, m, o, op) ({ \
__auto_type _v = _os_atomic_value_cast(p, v); \
_os_atomic_c11_op_orig(p, _v, m, o) op _v; \
})
#define _os_atomic_clang_op_orig(p, v, m, o) ({ \
_os_compiler_barrier_before_atomic(m); \
__auto_type _r = __atomic_##o(os_cast_to_nonatomic_pointer(p), \
_os_atomic_value_cast(p, v), \
_os_atomic_mo_##m##_smp); \
_os_compiler_barrier_after_atomic(m); \
_r; \
})
#define _os_atomic_clang_op(p, v, m, o, op) ({ \
__auto_type _v = _os_atomic_value_cast(p, v); \
__auto_type _s = _os_atomic_clang_op_orig(p, _v, m, o); \
op(_s, _v); \
})
#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#define _os_atomic_auto_dependency(e) \
_Generic(e, \
os_atomic_dependency_t: (e), \
default: os_atomic_make_dependency(e))
#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
#endif