/*
* @APPLE_LICENSE_HEADER_START@
*
* Copyright (c) 2011 Apple Inc. All Rights Reserved.
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this
* file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
/********************************************************************
*
* objc-msg-arm64.s - ARM64 code to support objc messaging
*
********************************************************************/
#ifdef __arm64__
#include <arm/arch.h>
#include "isa.h"
#include "arm64-asm.h"
.data
// _objc_entryPoints and _objc_exitPoints are used by method dispatch
// caching code to figure out whether any threads are actively
// in the cache for dispatching. The labels surround the asm code
// that do cache lookups. The tables are zero-terminated.
.align 4
.private_extern _objc_entryPoints
_objc_entryPoints:
PTR _cache_getImp
PTR _objc_msgSend
PTR _objc_msgSendSuper
PTR _objc_msgSendSuper2
PTR _objc_msgLookup
PTR _objc_msgLookupSuper2
PTR 0
.private_extern _objc_exitPoints
_objc_exitPoints:
PTR LExit_cache_getImp
PTR LExit_objc_msgSend
PTR LExit_objc_msgSendSuper
PTR LExit_objc_msgSendSuper2
PTR LExit_objc_msgLookup
PTR LExit_objc_msgLookupSuper2
PTR 0
/* objc_super parameter to sendSuper */
#define RECEIVER 0
#define CLASS __SIZEOF_POINTER__
/* Selected field offsets in class structure */
#define SUPERCLASS __SIZEOF_POINTER__
#define CACHE (2 * __SIZEOF_POINTER__)
/* Selected field offsets in method structure */
#define METHOD_NAME 0
#define METHOD_TYPES __SIZEOF_POINTER__
#define METHOD_IMP (2 * __SIZEOF_POINTER__)
#define BUCKET_SIZE (2 * __SIZEOF_POINTER__)
/********************************************************************
* GetClassFromIsa_p16 src
* src is a raw isa field. Sets p16 to the corresponding class pointer.
* The raw isa might be an indexed isa to be decoded, or a
* packed isa that needs to be masked.
*
* On exit:
* $0 is unchanged
* p16 is a class pointer
* x10 is clobbered
********************************************************************/
#if SUPPORT_INDEXED_ISA
.align 3
.globl _objc_indexed_classes
_objc_indexed_classes:
.fill ISA_INDEX_COUNT, PTRSIZE, 0
#endif
.macro GetClassFromIsa_p16 /* src */
#if SUPPORT_INDEXED_ISA
// Indexed isa
mov p16, $0 // optimistically set dst = src
tbz p16, #ISA_INDEX_IS_NPI_BIT, 1f // done if not non-pointer isa
// isa in p16 is indexed
adrp x10, _objc_indexed_classes@PAGE
add x10, x10, _objc_indexed_classes@PAGEOFF
ubfx p16, p16, #ISA_INDEX_SHIFT, #ISA_INDEX_BITS // extract index
ldr p16, [x10, p16, UXTP #PTRSHIFT] // load class from array
1:
#elif __LP64__
// 64-bit packed isa
and p16, $0, #ISA_MASK
#else
// 32-bit raw isa
mov p16, $0
#endif
.endmacro
/********************************************************************
* ENTRY functionName
* STATIC_ENTRY functionName
* END_ENTRY functionName
********************************************************************/
.macro ENTRY /* name */
.text
.align 5
.globl $0
$0:
.endmacro
.macro STATIC_ENTRY /*name*/
.text
.align 5
.private_extern $0
$0:
.endmacro
.macro END_ENTRY /* name */
LExit$0:
.endmacro
/********************************************************************
* UNWIND name, flags
* Unwind info generation
********************************************************************/
.macro UNWIND
.section __LD,__compact_unwind,regular,debug
PTR $0
.set LUnwind$0, LExit$0 - $0
.long LUnwind$0
.long $1
PTR 0 /* no personality */
PTR 0 /* no LSDA */
.text
.endmacro
#define NoFrame 0x02000000 // no frame, no SP adjustment
#define FrameWithNoSaves 0x04000000 // frame, no non-volatile saves
/********************************************************************
*
* CacheLookup NORMAL|GETIMP|LOOKUP
*
* Locate the implementation for a selector in a class method cache.
*
* Takes:
* x1 = selector
* x16 = class to be searched
*
* Kills:
* x9,x10,x11,x12, x17
*
* On exit: (found) calls or returns IMP
* with x16 = class, x17 = IMP
* (not found) jumps to LCacheMiss
*
********************************************************************/
#define NORMAL 0
#define GETIMP 1
#define LOOKUP 2
// CacheHit: x17 = cached IMP, x12 = address of cached IMP
.macro CacheHit
.if $0 == NORMAL
TailCallCachedImp x17, x12 // authenticate and call imp
.elseif $0 == GETIMP
mov p0, p17
AuthAndResignAsIMP x0, x12 // authenticate imp and re-sign as IMP
ret // return IMP
.elseif $0 == LOOKUP
AuthAndResignAsIMP x17, x12 // authenticate imp and re-sign as IMP
ret // return imp via x17
.else
.abort oops
.endif
.endmacro
.macro CheckMiss
// miss if bucket->sel == 0
.if $0 == GETIMP
cbz p9, LGetImpMiss
.elseif $0 == NORMAL
cbz p9, __objc_msgSend_uncached
.elseif $0 == LOOKUP
cbz p9, __objc_msgLookup_uncached
.else
.abort oops
.endif
.endmacro
.macro JumpMiss
.if $0 == GETIMP
b LGetImpMiss
.elseif $0 == NORMAL
b __objc_msgSend_uncached
.elseif $0 == LOOKUP
b __objc_msgLookup_uncached
.else
.abort oops
.endif
.endmacro
.macro CacheLookup
// p1 = SEL, p16 = isa
ldp p10, p11, [x16, #CACHE] // p10 = buckets, p11 = occupied|mask
#if !__LP64__
and w11, w11, 0xffff // p11 = mask
#endif
and w12, w1, w11 // x12 = _cmd & mask
add p12, p10, p12, LSL #(1+PTRSHIFT)
// p12 = buckets + ((_cmd & mask) << (1+PTRSHIFT))
ldp p17, p9, [x12] // {imp, sel} = *bucket
1: cmp p9, p1 // if (bucket->sel != _cmd)
b.ne 2f // scan more
CacheHit $0 // call or return imp
2: // not hit: p12 = not-hit bucket
CheckMiss $0 // miss if bucket->sel == 0
cmp p12, p10 // wrap if bucket == buckets
b.eq 3f
ldp p17, p9, [x12, #-BUCKET_SIZE]! // {imp, sel} = *--bucket
b 1b // loop
3: // wrap: p12 = first bucket, w11 = mask
add p12, p12, w11, UXTW #(1+PTRSHIFT)
// p12 = buckets + (mask << 1+PTRSHIFT)
// Clone scanning loop to miss instead of hang when cache is corrupt.
// The slow path may detect any corruption and halt later.
ldp p17, p9, [x12] // {imp, sel} = *bucket
1: cmp p9, p1 // if (bucket->sel != _cmd)
b.ne 2f // scan more
CacheHit $0 // call or return imp
2: // not hit: p12 = not-hit bucket
CheckMiss $0 // miss if bucket->sel == 0
cmp p12, p10 // wrap if bucket == buckets
b.eq 3f
ldp p17, p9, [x12, #-BUCKET_SIZE]! // {imp, sel} = *--bucket
b 1b // loop
3: // double wrap
JumpMiss $0
.endmacro
/********************************************************************
*
* id objc_msgSend(id self, SEL _cmd, ...) *
* objc_msgLookup ABI:
* IMP returned in x17
* x16 reserved for our use but not used
*
********************************************************************/
#if SUPPORT_TAGGED_POINTERS
.data
.align 3
.globl _objc_debug_taggedpointer_classes
_objc_debug_taggedpointer_classes:
.fill 16, 8, 0
.globl _objc_debug_taggedpointer_ext_classes
_objc_debug_taggedpointer_ext_classes:
.fill 256, 8, 0
#endif
ENTRY _objc_msgSend
UNWIND _objc_msgSend, NoFrame
cmp p0, #0 // nil check and tagged pointer check
#if SUPPORT_TAGGED_POINTERS
b.le LNilOrTagged // (MSB tagged pointer looks negative)
#else
b.eq LReturnZero
#endif
ldr p13, [x0] // p13 = isa
GetClassFromIsa_p16 p13 // p16 = class
LGetIsaDone:
CacheLookup NORMAL // calls imp or objc_msgSend_uncached
#if SUPPORT_TAGGED_POINTERS
LNilOrTagged:
b.eq LReturnZero // nil check
// tagged
adrp x10, _objc_debug_taggedpointer_classes@PAGE
add x10, x10, _objc_debug_taggedpointer_classes@PAGEOFF
ubfx x11, x0, #60, #4
ldr x16, [x10, x11, LSL #3]
adrp x10, _OBJC_CLASS_$___NSUnrecognizedTaggedPointer@PAGE
add x10, x10, _OBJC_CLASS_$___NSUnrecognizedTaggedPointer@PAGEOFF
cmp x10, x16
b.ne LGetIsaDone
// ext tagged
adrp x10, _objc_debug_taggedpointer_ext_classes@PAGE
add x10, x10, _objc_debug_taggedpointer_ext_classes@PAGEOFF
ubfx x11, x0, #52, #8
ldr x16, [x10, x11, LSL #3]
b LGetIsaDone
// SUPPORT_TAGGED_POINTERS
#endif
LReturnZero:
// x0 is already zero
mov x1, #0
movi d0, #0
movi d1, #0
movi d2, #0
movi d3, #0
ret
END_ENTRY _objc_msgSend
ENTRY _objc_msgLookup
UNWIND _objc_msgLookup, NoFrame
cmp p0, #0 // nil check and tagged pointer check
#if SUPPORT_TAGGED_POINTERS
b.le LLookup_NilOrTagged // (MSB tagged pointer looks negative)
#else
b.eq LLookup_Nil
#endif
ldr p13, [x0] // p13 = isa
GetClassFromIsa_p16 p13 // p16 = class
LLookup_GetIsaDone:
CacheLookup LOOKUP // returns imp
#if SUPPORT_TAGGED_POINTERS
LLookup_NilOrTagged:
b.eq LLookup_Nil // nil check
// tagged
mov x10, #0xf000000000000000
cmp x0, x10
b.hs LLookup_ExtTag
adrp x10, _objc_debug_taggedpointer_classes@PAGE
add x10, x10, _objc_debug_taggedpointer_classes@PAGEOFF
ubfx x11, x0, #60, #4
ldr x16, [x10, x11, LSL #3]
b LLookup_GetIsaDone
LLookup_ExtTag:
adrp x10, _objc_debug_taggedpointer_ext_classes@PAGE
add x10, x10, _objc_debug_taggedpointer_ext_classes@PAGEOFF
ubfx x11, x0, #52, #8
ldr x16, [x10, x11, LSL #3]
b LLookup_GetIsaDone
// SUPPORT_TAGGED_POINTERS
#endif
LLookup_Nil:
adrp x17, __objc_msgNil@PAGE
add x17, x17, __objc_msgNil@PAGEOFF
ret
END_ENTRY _objc_msgLookup
STATIC_ENTRY __objc_msgNil
// x0 is already zero
mov x1, #0
movi d0, #0
movi d1, #0
movi d2, #0
movi d3, #0
ret
END_ENTRY __objc_msgNil
ENTRY _objc_msgSendSuper
UNWIND _objc_msgSendSuper, NoFrame
ldp p0, p16, [x0] // p0 = real receiver, p16 = class
CacheLookup NORMAL // calls imp or objc_msgSend_uncached
END_ENTRY _objc_msgSendSuper
// no _objc_msgLookupSuper
ENTRY _objc_msgSendSuper2
UNWIND _objc_msgSendSuper2, NoFrame
ldp p0, p16, [x0] // p0 = real receiver, p16 = class
ldr p16, [x16, #SUPERCLASS] // p16 = class->superclass
CacheLookup NORMAL
END_ENTRY _objc_msgSendSuper2
ENTRY _objc_msgLookupSuper2
UNWIND _objc_msgLookupSuper2, NoFrame
ldp p0, p16, [x0] // p0 = real receiver, p16 = class
ldr p16, [x16, #SUPERCLASS] // p16 = class->superclass
CacheLookup LOOKUP
END_ENTRY _objc_msgLookupSuper2
.macro MethodTableLookup
// push frame
SignLR
stp fp, lr, [sp, #-16]!
mov fp, sp
// save parameter registers: x0..x8, q0..q7
sub sp, sp, #(10*8 + 8*16)
stp q0, q1, [sp, #(0*16)]
stp q2, q3, [sp, #(2*16)]
stp q4, q5, [sp, #(4*16)]
stp q6, q7, [sp, #(6*16)]
stp x0, x1, [sp, #(8*16+0*8)]
stp x2, x3, [sp, #(8*16+2*8)]
stp x4, x5, [sp, #(8*16+4*8)]
stp x6, x7, [sp, #(8*16+6*8)]
str x8, [sp, #(8*16+8*8)]
// receiver and selector already in x0 and x1
mov x2, x16
bl __class_lookupMethodAndLoadCache3
// IMP in x0
mov x17, x0
// restore registers and return
ldp q0, q1, [sp, #(0*16)]
ldp q2, q3, [sp, #(2*16)]
ldp q4, q5, [sp, #(4*16)]
ldp q6, q7, [sp, #(6*16)]
ldp x0, x1, [sp, #(8*16+0*8)]
ldp x2, x3, [sp, #(8*16+2*8)]
ldp x4, x5, [sp, #(8*16+4*8)]
ldp x6, x7, [sp, #(8*16+6*8)]
ldr x8, [sp, #(8*16+8*8)]
mov sp, fp
ldp fp, lr, [sp], #16
AuthenticateLR
.endmacro
STATIC_ENTRY __objc_msgSend_uncached
UNWIND __objc_msgSend_uncached, FrameWithNoSaves
// THIS IS NOT A CALLABLE C FUNCTION
// Out-of-band p16 is the class to search
MethodTableLookup
TailCallFunctionPointer x17
END_ENTRY __objc_msgSend_uncached
STATIC_ENTRY __objc_msgLookup_uncached
UNWIND __objc_msgLookup_uncached, FrameWithNoSaves
// THIS IS NOT A CALLABLE C FUNCTION
// Out-of-band p16 is the class to search
MethodTableLookup
ret
END_ENTRY __objc_msgLookup_uncached
STATIC_ENTRY _cache_getImp
GetClassFromIsa_p16 p0
CacheLookup GETIMP
LGetImpMiss:
mov p0, #0
ret
END_ENTRY _cache_getImp
/********************************************************************
*
* id _objc_msgForward(id self, SEL _cmd,...)* _objc_msgForward is the externally-callable
* function returned by things like method_getImplementation().
* _objc_msgForward_impcache is the function pointer actually stored in
* method caches.
*
********************************************************************/
STATIC_ENTRY __objc_msgForward_impcache
// No stret specialization.
b __objc_msgForward
END_ENTRY __objc_msgForward_impcache
ENTRY __objc_msgForward
adrp x17, __objc_forward_handler@PAGE
ldr p17, [x17, __objc_forward_handler@PAGEOFF]
TailCallFunctionPointer x17
END_ENTRY __objc_msgForward
ENTRY _objc_msgSend_noarg
b _objc_msgSend
END_ENTRY _objc_msgSend_noarg
ENTRY _objc_msgSend_debug
b _objc_msgSend
END_ENTRY _objc_msgSend_debug
ENTRY _objc_msgSendSuper2_debug
b _objc_msgSendSuper2
END_ENTRY _objc_msgSendSuper2_debug
ENTRY _method_invoke
// x1 is method triplet instead of SEL
add p16, p1, #METHOD_IMP
ldr p17, [x16]
ldr p1, [x1, #METHOD_NAME]
TailCallMethodListImp x17, x16
END_ENTRY _method_invoke
#endif