/*
* Copyright (c) 2007 Apple Inc. All rights reserved.
* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this
* file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
#include <machine/cpu_capabilities.h>
#include "platfunc.h"
#include <architecture/i386/asm_help.h>
#define DECLARE(x) \
.align 2, 0x90 .globl x ## Barrier x ## Barrier:
.text
#define ATOMIC_UP 0
#define ATOMIC_MP 1
#define ATOMIC_RET_ORIG 0
#define ATOMIC_RET_NEW 1
// compare and exchange 32-bit
// xchg32 <new> <dst> <mp>
.macro xchg32
.if $2 == ATOMIC_MP
lock
.endif
cmpxchgl $0, ($1)
.endm
// xchg64 <new> <dst> <mp>
.macro xchg64
.if $2 == ATOMIC_MP
lock
.endif
cmpxchg $0, ($1)
.endm
#define ATOMIC_ARITHMETIC(instr, orig, mp) \
movl (%rsi), %eax /* get 2nd arg -> eax */ instr %edi, %edx /* apply instr to %edx with arg2 */ jnz 1b /* jump if failed */ movl %edx, %eax /* return the new value */
// Used in OSAtomicTestAndSet( uint32_t n, void *value ), assumes ABI parameter loctions
// Manpage says bit to test/set is (0x80 >> (n & 7)) of byte (addr + (n >> 3))
#define ATOMIC_BIT_OP(instr, mp) \
xorl $7, %edi /* bit position is numbered big endian so convert to little endian */ addq %rdi, %rsi /* generate bit address */ andq $31, %rdi /* keep bit offset in range 0..31 */ shrq $3, %rsi /* get 4-byte aligned address */ lock /* lock the bit test */ instr %edi, (%rsi) /* do the bit test, supplied into the macro */ movzbl %al,%eax /* widen in case caller assumes we return an int */
// uint32_t OSAtomicAnd32( uint32_t mask, uint32_t *value)PLATFUNC_FUNCTION_START(OSAtomicAnd32Barrier, up, 64, 2)
ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Barrier, mp, 64, 2)
ATOMIC_ARITHMETIC(andl, ATOMIC_RET_NEW, ATOMIC_MP)
ret
// uint32_t OSAtomicOr32( uint32_t mask, uint32_t *value)PLATFUNC_FUNCTION_START(OSAtomicOr32Barrier, up, 64, 2)
ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Barrier, mp, 64, 2)
ATOMIC_ARITHMETIC(orl, ATOMIC_RET_NEW, ATOMIC_MP)
ret
// uint32_t OSAtomicXor32( uint32_t mask, uint32_t *value)PLATFUNC_FUNCTION_START(OSAtomicXor32Barrier, up, 64, 2)
ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Barrier, mp, 64, 2)
ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_NEW, ATOMIC_MP)
ret
// uint32_t OSAtomicAnd32Orig( uint32_t mask, uint32_t *value)PLATFUNC_FUNCTION_START(OSAtomicAnd32OrigBarrier, up, 64, 2)
ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAnd32OrigBarrier, mp, 64, 2)
ATOMIC_ARITHMETIC(andl, ATOMIC_RET_ORIG, ATOMIC_MP)
ret
// uint32_t OSAtomicOr32Orig( uint32_t mask, uint32_t *value)PLATFUNC_FUNCTION_START(OSAtomicOr32OrigBarrier, up, 64, 2)
ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicOr32OrigBarrier, mp, 64, 2)
ATOMIC_ARITHMETIC(orl, ATOMIC_RET_ORIG, ATOMIC_MP)
ret
// uint32_t OSAtomicXor32Orig( uint32_t mask, uint32_t *value)PLATFUNC_FUNCTION_START(OSAtomicXor32OrigBarrier, up, 64, 2)
ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32Orig, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicXor32OrigBarrier, mp, 64, 2)
ATOMIC_ARITHMETIC(xorl, ATOMIC_RET_ORIG, ATOMIC_MP)
ret
// bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value)PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapIntBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap32Barrier, up, 64, 2)
movl %edi, %eax
xchg32 %esi, %rdx, ATOMIC_UP
sete %al
movzbl %al,%eax // widen in case caller assumes we return an int
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapInt, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapIntBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap32Barrier, mp, 64, 2)
movl %edi, %eax
xchg32 %esi, %rdx, ATOMIC_MP
sete %al
movzbl %al,%eax // widen in case caller assumes we return an int
ret
// bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value)PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapPtrBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLong, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwapLongBarrier, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64, up, 64, 2)
PLATFUNC_FUNCTION_START(OSAtomicCompareAndSwap64Barrier, up, 64, 2)
mov %rdi, %rax
xchg64 %rsi, %rdx, ATOMIC_UP
sete %al
movzbl %al,%eax // widen in case caller assumes we return an int
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtr, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapPtrBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLong, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwapLongBarrier, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicCompareAndSwap64Barrier, mp, 64, 2)
mov %rdi, %rax
xchg64 %rsi, %rdx, ATOMIC_MP
sete %al
movzbl %al,%eax // widen in case caller assumes we return an int
ret
// int32_t OSAtomicAdd32( int32_t amt, int32_t *value )PLATFUNC_FUNCTION_START(OSAtomicAdd32Barrier, up, 64, 2)
movl %edi, %eax // save amt to add
xaddl %edi, (%rsi) // swap and add value, returns old value in %edi
addl %edi, %eax // add old value to amt as return value
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd32Barrier, mp, 64, 2)
movl %edi, %eax // save amt to add
lock // lock prefix breaks tabs addl %edi, %eax // add old value to amt as return value
ret
// int64_t OSAtomicAdd64( int64_t amt, int64_t *value )PLATFUNC_FUNCTION_START(OSAtomicAdd64Barrier, up, 64, 2)
movq %rdi, %rax // save amt to add
xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
addq %rdi, %rax // add old value to amt as return value
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicAdd64Barrier, mp, 64, 2)
movq %rdi, %rax // save amt to add
lock
xadd %rdi, (%rsi) // swap and add value, returns old value in %rsi
addq %rdi, %rax // add old value to amt as return value
ret
// bool OSAtomicTestAndSet( uint32_t n, void *value )PLATFUNC_FUNCTION_START(OSAtomicTestAndSetBarrier, up, 64, 2)
ATOMIC_BIT_OP(btsl, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSet, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndSetBarrier, mp, 64, 2)
ATOMIC_BIT_OP(btsl, ATOMIC_MP)
ret
// bool OSAtomicTestAndClear( uint32_t n, void *value )PLATFUNC_FUNCTION_START(OSAtomicTestAndClearBarrier, up, 64, 2)
ATOMIC_BIT_OP(btrl, ATOMIC_UP)
ret
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClear, mp, 64, 2)
PLATFUNC_FUNCTION_START_GENERIC(OSAtomicTestAndClearBarrier, mp, 64, 2)
ATOMIC_BIT_OP(btrl, ATOMIC_MP)
ret
// void OSMemoryBarrier( void ) .globl _OSMemoryBarrier
_OSMemoryBarrier:
mfence
ret
/*
* typedef volatile struct {
* void *opaque1 * } OSQueueHead * void OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset) .align 2
.globl _OSAtomicEnqueue
_OSAtomicEnqueue: // %rdi == list head, %rsi == new, %rdx == offset
pushq %rbx
movq %rsi,%rbx // %rbx == new
movq %rdx,%rsi // %rsi == offset
movq (%rdi),%rax // %rax == ptr to 1st element in Q
movq 8(%rdi),%rdx // %rdx == current generation count
1:
movq %rax,(%rbx,%rsi)// link to old list head from new element
movq %rdx,%rcx
incq %rcx // increment generation count
lock // always lock for now...
cmpxchg16b (%rdi) // ...push on new element
jnz 1b
popq %rbx
ret
/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset) .globl _OSAtomicDequeue
_OSAtomicDequeue: // %rdi == list head, %rsi == offset
pushq %rbx
movq (%rdi),%rax // %rax == ptr to 1st element in Q
movq 8(%rdi),%rdx // %rdx == current generation count
1:
testq %rax,%rax // list empty?
jz 2f // yes
movq (%rax,%rsi),%rbx // point to 2nd in Q
movq %rdx,%rcx
incq %rcx // increment generation count
lock // always lock for now...
cmpxchg16b (%rdi) // ...pop off 1st element
jnz 1b
2:
popq %rbx
ret // ptr to 1st element in Q still in %rax
/*
* typedef volatile struct {
* void *opaque1 * int opaque3 *
* void OSAtomicFifoEnqueue( OSFifoQueueHead *list, void *new, size_t offset) .align 2
.globl _OSAtomicFifoEnqueue
_OSAtomicFifoEnqueue:
pushq %rbx
xorl %ebx,%ebx // clear "preemption pending" flag
movq _commpage_pfz_base(%rip),%rcx
addq $(_COMM_TEXT_PFZ_ENQUEUE_OFFSET), %rcx
call *%rcx
testl %ebx,%ebx // pending preemption?
jz 1f
call _preempt // call into the kernel to pfz_exit
1:
popq %rbx
ret
/* void* OSAtomicFifoDequeue( OSFifoQueueHead *list, size_t offset) .globl _OSAtomicFifoDequeue
_OSAtomicFifoDequeue:
pushq %rbx
xorl %ebx,%ebx // clear "preemption pending" flag
movq _commpage_pfz_base(%rip), %rcx
movq %rsi,%rdx // move offset to %rdx to be like the Enqueue case
addq $(_COMM_TEXT_PFZ_DEQUEUE_OFFSET), %rcx
call *%rcx
testl %ebx,%ebx // pending preemption?
jz 1f
call _preempt // call into the kernel to pfz_exit
1:
popq %rbx
ret // ptr to 1st element in Q in %rax
// Local Variables:
// tab-width: 8
// End: