/*
* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#include <sys/appleapiopts.h>
#include <ppc/asm.h> // EXT, LEXT
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
/* OSAtomic.h library native implementations. */
.text
.align 2
atomic_add32: // int32_t OSAtomicAdd32( int32_t amt, int32_t *value ) lwarx r5,0,r4
add r6,r3,r5
stwcx. r6,0,r4
bne-- 1b
mr r3,r6
blr
COMMPAGE_DESCRIPTOR(atomic_add32,_COMM_PAGE_ATOMIC_ADD32,0,0,kCommPageBoth)
atomic_add64: // int64_t OSAtomicAdd64( int64_t amt, int64_t *value ) ldarx r5,0,r4
add r6,r3,r5
stdcx. r6,0,r4
bne-- 1b
mr r3,r6
blr
COMMPAGE_DESCRIPTOR(atomic_add64,_COMM_PAGE_ATOMIC_ADD64,k64Bit,0,kCommPage64)
/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This is the no-barrier version */
compare_and_swap32_on32: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value) lwarx r7,0,r5
cmplw r7,r3
bne- 2f
stwcx. r4,0,r5
bne- 1b
li r3,1
blr
2:
li r3,0 // return failure
blr
COMMPAGE_DESCRIPTOR(compare_and_swap32_on32,_COMM_PAGE_COMPARE_AND_SWAP32,0,k64Bit,kCommPageBoth)
/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This is the no-barrier version */
compare_and_swap32_on64: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value) lwarx r7,0,r5
cmplw r7,r3
bne-- 2f
stwcx. r4,0,r5
bne-- 1b
li r3,1
blr
2:
li r8,-8 // on 970, must release reservation
li r3,0 // return failure
stwcx. r4,r8,r1 // store into red zone to release
blr
COMMPAGE_DESCRIPTOR(compare_and_swap32_on64,_COMM_PAGE_COMPARE_AND_SWAP32,k64Bit,0,kCommPageBoth)
/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This is the no-barrier version */
compare_and_swap64: // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value) ldarx r7,0,r5
cmpld r7,r3
bne-- 2f
stdcx. r4,0,r5
bne-- 1b
li r3,1
blr
2:
li r8,-8 // on 970, must release reservation
li r3,0 // return failure
stdcx. r4,r8,r1 // store into red zone to release
blr
COMMPAGE_DESCRIPTOR(compare_and_swap64,_COMM_PAGE_COMPARE_AND_SWAP64,k64Bit,0,kCommPage64)
/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This version of compare-and-swap incorporates a memory barrier. */
compare_and_swap32_on32b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value)1:
lwarx r7,0,r5
cmplw r7,r3
bne- 2f
stwcx. r4,0,r5
bne- 1b
isync // read barrier, NOP'd on a UP
li r3,1
blr
2:
li r3,0 // return failure
blr
COMMPAGE_DESCRIPTOR(compare_and_swap32_on32b,_COMM_PAGE_COMPARE_AND_SWAP32B,0,k64Bit,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)
/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This version of compare-and-swap incorporates a memory barrier. */
compare_and_swap32_on64b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value)1:
lwarx r7,0,r5
cmplw r7,r3
bne-- 2f
stwcx. r4,0,r5
bne-- 1b
isync // read barrier, NOP'd on a UP
li r3,1
blr
2:
li r8,-8 // on 970, must release reservation
li r3,0 // return failure
stwcx. r4,r8,r1 // store into red zone to release
blr
COMMPAGE_DESCRIPTOR(compare_and_swap32_on64b,_COMM_PAGE_COMPARE_AND_SWAP32B,k64Bit,0,kCommPageBoth+kCommPageSYNC+kCommPageISYNC)
/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */
/* This version of compare-and-swap incorporates a memory barrier. */
compare_and_swap64b: // bool OSAtomicCompareAndSwapBarrier64( int64_t old, int64_t new, int64_t *value)1:
ldarx r7,0,r5
cmpld r7,r3
bne-- 2f
stdcx. r4,0,r5
bne-- 1b
isync // read barrier, NOP'd on a UP
li r3,1
blr
2:
li r8,-8 // on 970, must release reservation
li r3,0 // return failure
stdcx. r4,r8,r1 // store into red zone to release
blr
COMMPAGE_DESCRIPTOR(compare_and_swap64b,_COMM_PAGE_COMPARE_AND_SWAP64B,k64Bit,0,kCommPage64+kCommPageSYNC+kCommPageISYNC)
atomic_enqueue32: // void OSAtomicEnqueue( void **list, void *new, size_t offset) lwarx r6,0,r3 // get link to 1st on list
stwx r6,r4,r5 // hang list off new node
eieio // make sure the "stwx" comes before "stwcx." (nop'd on UP)
stwcx. r4,0,r3 // make new 1st on list
beqlr++
b 1b
COMMPAGE_DESCRIPTOR(atomic_enqueue32,_COMM_PAGE_ENQUEUE,0,0,kCommPageSYNC+kCommPage32)
atomic_enqueue64: // void OSAtomicEnqueue( void **list, void *new, size_t offset) ldarx r6,0,r3 // get link to 1st on list
stdx r6,r4,r5 // hang list off new node
lwsync // make sure the "stdx" comes before the "stdcx." (nop'd on UP)
stdcx. r4,0,r3 // make new 1st on list
beqlr++
b 1b
COMMPAGE_DESCRIPTOR(atomic_enqueue64,_COMM_PAGE_ENQUEUE,k64Bit,0,kCommPageSYNC+kCommPage64)
atomic_dequeue32_on32: // void* OSAtomicDequeue( void **list, size_t offset)1:
lwarx r3,0,r5 // get 1st in list
cmpwi r3,0 // null?
beqlr // yes, list empty
lwzx r6,r3,r4 // get 2nd
stwcx. r6,0,r5 // make 2nd first
bne-- 1b
isync // cancel read-aheads (nop'd on UP)
blr
COMMPAGE_DESCRIPTOR(atomic_dequeue32_on32,_COMM_PAGE_DEQUEUE,0,k64Bit,kCommPageISYNC+kCommPage32)
atomic_dequeue32_on64: // void* OSAtomicDequeue( void **list, size_t offset) li r7,-8 // use red zone to release reservation if necessary
1:
lwarx r3,0,r5 // get 1st in list
cmpwi r3,0 // null?
beq 2f
lwzx r6,r3,r4 // get 2nd
stwcx. r6,0,r5 // make 2nd first
isync // cancel read-aheads (nop'd on UP)
beqlr++ // return next element in r2
b 1b // retry (lost reservation)
2:
stwcx. r0,r7,r1 // on 970, release reservation using red zone
blr // return null
COMMPAGE_DESCRIPTOR(atomic_dequeue32_on64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage32)
atomic_dequeue64: // void* OSAtomicDequeue( void **list, size_t offset) li r7,-8 // use red zone to release reservation if necessary
1:
ldarx r3,0,r5 // get 1st in list
cmpdi r3,0 // null?
beq 2f
ldx r6,r3,r4 // get 2nd
stdcx. r6,0,r5 // make 2nd first
isync // cancel read-aheads (nop'd on UP)
beqlr++ // return next element in r2
b 1b // retry (lost reservation)
2:
stdcx. r0,r7,r1 // on 970, release reservation using red zone
blr // return null
COMMPAGE_DESCRIPTOR(atomic_dequeue64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage64)
memory_barrier_up: // void OSMemoryBarrier( void )
blr // nothing to do on UP
COMMPAGE_DESCRIPTOR(memory_barrier_up,_COMM_PAGE_MEMORY_BARRIER,kUP,0,kCommPageBoth)
memory_barrier_mp32: // void OSMemoryBarrier( void )
isync // we use eieio in preference to sync...
eieio // ...because it is faster
blr
COMMPAGE_DESCRIPTOR(memory_barrier_mp32,_COMM_PAGE_MEMORY_BARRIER,0,kUP+k64Bit,kCommPage32)
memory_barrier_mp64: // void OSMemoryBarrier( void )
isync
lwsync // on 970, lwsync is faster than eieio
blr
COMMPAGE_DESCRIPTOR(memory_barrier_mp64,_COMM_PAGE_MEMORY_BARRIER,k64Bit,kUP,kCommPageBoth)