#ifndef _OBJC_OBJCOBJECT_H_
#define _OBJC_OBJCOBJECT_H_
#include "objc-private.h"
enum ReturnDisposition : bool {
ReturnAtPlus0 = false, ReturnAtPlus1 = true
};
static ALWAYS_INLINE
bool prepareOptimizedReturn(ReturnDisposition disposition);
#if SUPPORT_TAGGED_POINTERS
extern "C" {
extern Class objc_debug_taggedpointer_classes[_OBJC_TAG_SLOT_COUNT*2];
extern Class objc_debug_taggedpointer_ext_classes[_OBJC_TAG_EXT_SLOT_COUNT];
}
#define objc_tag_classes objc_debug_taggedpointer_classes
#define objc_tag_ext_classes objc_debug_taggedpointer_ext_classes
#endif
#if SUPPORT_INDEXED_ISA
ALWAYS_INLINE Class &
classForIndex(uintptr_t index) {
assert(index > 0);
assert(index < (uintptr_t)objc_indexed_classes_count);
return objc_indexed_classes[index];
}
#endif
inline bool
objc_object::isClass()
{
if (isTaggedPointer()) return false;
return ISA()->isMetaClass();
}
#if SUPPORT_TAGGED_POINTERS
inline Class
objc_object::getIsa()
{
if (!isTaggedPointer()) return ISA();
uintptr_t ptr = (uintptr_t)this;
if (isExtTaggedPointer()) {
uintptr_t slot =
(ptr >> _OBJC_TAG_EXT_SLOT_SHIFT) & _OBJC_TAG_EXT_SLOT_MASK;
return objc_tag_ext_classes[slot];
} else {
uintptr_t slot =
(ptr >> _OBJC_TAG_SLOT_SHIFT) & _OBJC_TAG_SLOT_MASK;
return objc_tag_classes[slot];
}
}
inline bool
objc_object::isTaggedPointer()
{
return _objc_isTaggedPointer(this);
}
inline bool
objc_object::isBasicTaggedPointer()
{
return isTaggedPointer() && !isExtTaggedPointer();
}
inline bool
objc_object::isExtTaggedPointer()
{
return ((uintptr_t)this & _OBJC_TAG_EXT_MASK) == _OBJC_TAG_EXT_MASK;
}
#else
inline Class
objc_object::getIsa()
{
return ISA();
}
inline bool
objc_object::isTaggedPointer()
{
return false;
}
inline bool
objc_object::isBasicTaggedPointer()
{
return false;
}
inline bool
objc_object::isExtTaggedPointer()
{
return false;
}
#endif
#if SUPPORT_NONPOINTER_ISA
inline Class
objc_object::ISA()
{
assert(!isTaggedPointer());
#if SUPPORT_INDEXED_ISA
if (isa.nonpointer) {
uintptr_t slot = isa.indexcls;
return classForIndex((unsigned)slot);
}
return (Class)isa.bits;
#else
return (Class)(isa.bits & ISA_MASK);
#endif
}
inline bool
objc_object::hasNonpointerIsa()
{
return isa.nonpointer;
}
inline void
objc_object::initIsa(Class cls)
{
initIsa(cls, false, false);
}
inline void
objc_object::initClassIsa(Class cls)
{
if (DisableNonpointerIsa || cls->instancesRequireRawIsa()) {
initIsa(cls, false, false);
} else {
initIsa(cls, true, false);
}
}
inline void
objc_object::initProtocolIsa(Class cls)
{
return initClassIsa(cls);
}
inline void
objc_object::initInstanceIsa(Class cls, bool hasCxxDtor)
{
assert(!cls->instancesRequireRawIsa());
assert(hasCxxDtor == cls->hasCxxDtor());
initIsa(cls, true, hasCxxDtor);
}
inline void
objc_object::initIsa(Class cls, bool nonpointer, bool hasCxxDtor)
{
assert(!isTaggedPointer());
if (!nonpointer) {
isa.cls = cls;
} else {
assert(!DisableNonpointerIsa);
assert(!cls->instancesRequireRawIsa());
isa_t newisa(0);
#if SUPPORT_INDEXED_ISA
assert(cls->classArrayIndex() > 0);
newisa.bits = ISA_INDEX_MAGIC_VALUE;
newisa.has_cxx_dtor = hasCxxDtor;
newisa.indexcls = (uintptr_t)cls->classArrayIndex();
#else
newisa.bits = ISA_MAGIC_VALUE;
newisa.has_cxx_dtor = hasCxxDtor;
newisa.shiftcls = (uintptr_t)cls >> 3;
#endif
isa = newisa;
}
}
inline Class
objc_object::changeIsa(Class newCls)
{
assert(!isTaggedPointer());
isa_t oldisa;
isa_t newisa;
bool sideTableLocked = false;
bool transcribeToSideTable = false;
do {
transcribeToSideTable = false;
oldisa = LoadExclusive(&isa.bits);
if ((oldisa.bits == 0 || oldisa.nonpointer) &&
!newCls->isFuture() && newCls->canAllocNonpointer())
{
#if SUPPORT_INDEXED_ISA
if (oldisa.bits == 0) newisa.bits = ISA_INDEX_MAGIC_VALUE;
else newisa = oldisa;
newisa.has_cxx_dtor = newCls->hasCxxDtor();
assert(newCls->classArrayIndex() > 0);
newisa.indexcls = (uintptr_t)newCls->classArrayIndex();
#else
if (oldisa.bits == 0) newisa.bits = ISA_MAGIC_VALUE;
else newisa = oldisa;
newisa.has_cxx_dtor = newCls->hasCxxDtor();
newisa.shiftcls = (uintptr_t)newCls >> 3;
#endif
}
else if (oldisa.nonpointer) {
if (!sideTableLocked) sidetable_lock();
sideTableLocked = true;
transcribeToSideTable = true;
newisa.cls = newCls;
}
else {
newisa.cls = newCls;
}
} while (!StoreExclusive(&isa.bits, oldisa.bits, newisa.bits));
if (transcribeToSideTable) {
sidetable_moveExtraRC_nolock(oldisa.extra_rc,
oldisa.deallocating,
oldisa.weakly_referenced);
}
if (sideTableLocked) sidetable_unlock();
if (oldisa.nonpointer) {
#if SUPPORT_INDEXED_ISA
return classForIndex(oldisa.indexcls);
#else
return (Class)((uintptr_t)oldisa.shiftcls << 3);
#endif
}
else {
return oldisa.cls;
}
}
inline bool
objc_object::hasAssociatedObjects()
{
if (isTaggedPointer()) return true;
if (isa.nonpointer) return isa.has_assoc;
return true;
}
inline void
objc_object::setHasAssociatedObjects()
{
if (isTaggedPointer()) return;
retry:
isa_t oldisa = LoadExclusive(&isa.bits);
isa_t newisa = oldisa;
if (!newisa.nonpointer || newisa.has_assoc) {
ClearExclusive(&isa.bits);
return;
}
newisa.has_assoc = true;
if (!StoreExclusive(&isa.bits, oldisa.bits, newisa.bits)) goto retry;
}
inline bool
objc_object::isWeaklyReferenced()
{
assert(!isTaggedPointer());
if (isa.nonpointer) return isa.weakly_referenced;
else return sidetable_isWeaklyReferenced();
}
inline void
objc_object::setWeaklyReferenced_nolock()
{
retry:
isa_t oldisa = LoadExclusive(&isa.bits);
isa_t newisa = oldisa;
if (slowpath(!newisa.nonpointer)) {
ClearExclusive(&isa.bits);
sidetable_setWeaklyReferenced_nolock();
return;
}
if (newisa.weakly_referenced) {
ClearExclusive(&isa.bits);
return;
}
newisa.weakly_referenced = true;
if (!StoreExclusive(&isa.bits, oldisa.bits, newisa.bits)) goto retry;
}
inline bool
objc_object::hasCxxDtor()
{
assert(!isTaggedPointer());
if (isa.nonpointer) return isa.has_cxx_dtor;
else return isa.cls->hasCxxDtor();
}
inline bool
objc_object::rootIsDeallocating()
{
if (isTaggedPointer()) return false;
if (isa.nonpointer) return isa.deallocating;
return sidetable_isDeallocating();
}
inline void
objc_object::clearDeallocating()
{
if (slowpath(!isa.nonpointer)) {
sidetable_clearDeallocating();
}
else if (slowpath(isa.weakly_referenced || isa.has_sidetable_rc)) {
clearDeallocating_slow();
}
assert(!sidetable_present());
}
inline void
objc_object::rootDealloc()
{
if (isTaggedPointer()) return;
if (fastpath(isa.nonpointer &&
!isa.weakly_referenced &&
!isa.has_assoc &&
!isa.has_cxx_dtor &&
!isa.has_sidetable_rc))
{
assert(!sidetable_present());
free(this);
}
else {
object_dispose((id)this);
}
}
inline id
objc_object::retain()
{
assert(!isTaggedPointer());
if (fastpath(!ISA()->hasCustomRR())) {
return rootRetain();
}
return ((id(*)(objc_object *, SEL))objc_msgSend)(this, SEL_retain);
}
ALWAYS_INLINE id
objc_object::rootRetain()
{
return rootRetain(false, false);
}
ALWAYS_INLINE bool
objc_object::rootTryRetain()
{
return rootRetain(true, false) ? true : false;
}
ALWAYS_INLINE id
objc_object::rootRetain(bool tryRetain, bool handleOverflow)
{
if (isTaggedPointer()) return (id)this;
bool sideTableLocked = false;
bool transcribeToSideTable = false;
isa_t oldisa;
isa_t newisa;
do {
transcribeToSideTable = false;
oldisa = LoadExclusive(&isa.bits);
newisa = oldisa;
if (slowpath(!newisa.nonpointer)) {
ClearExclusive(&isa.bits);
if (!tryRetain && sideTableLocked) sidetable_unlock();
if (tryRetain) return sidetable_tryRetain() ? (id)this : nil;
else return sidetable_retain();
}
if (slowpath(tryRetain && newisa.deallocating)) {
ClearExclusive(&isa.bits);
if (!tryRetain && sideTableLocked) sidetable_unlock();
return nil;
}
uintptr_t carry;
newisa.bits = addc(newisa.bits, RC_ONE, 0, &carry);
if (slowpath(carry)) {
if (!handleOverflow) {
ClearExclusive(&isa.bits);
return rootRetain_overflow(tryRetain);
}
if (!tryRetain && !sideTableLocked) sidetable_lock();
sideTableLocked = true;
transcribeToSideTable = true;
newisa.extra_rc = RC_HALF;
newisa.has_sidetable_rc = true;
}
} while (slowpath(!StoreExclusive(&isa.bits, oldisa.bits, newisa.bits)));
if (slowpath(transcribeToSideTable)) {
sidetable_addExtraRC_nolock(RC_HALF);
}
if (slowpath(!tryRetain && sideTableLocked)) sidetable_unlock();
return (id)this;
}
inline void
objc_object::release()
{
assert(!isTaggedPointer());
if (fastpath(!ISA()->hasCustomRR())) {
rootRelease();
return;
}
((void(*)(objc_object *, SEL))objc_msgSend)(this, SEL_release);
}
ALWAYS_INLINE bool
objc_object::rootRelease()
{
return rootRelease(true, false);
}
ALWAYS_INLINE bool
objc_object::rootReleaseShouldDealloc()
{
return rootRelease(false, false);
}
ALWAYS_INLINE bool
objc_object::rootRelease(bool performDealloc, bool handleUnderflow)
{
if (isTaggedPointer()) return false;
bool sideTableLocked = false;
isa_t oldisa;
isa_t newisa;
retry:
do {
oldisa = LoadExclusive(&isa.bits);
newisa = oldisa;
if (slowpath(!newisa.nonpointer)) {
ClearExclusive(&isa.bits);
if (sideTableLocked) sidetable_unlock();
return sidetable_release(performDealloc);
}
uintptr_t carry;
newisa.bits = subc(newisa.bits, RC_ONE, 0, &carry); if (slowpath(carry)) {
goto underflow;
}
} while (slowpath(!StoreReleaseExclusive(&isa.bits,
oldisa.bits, newisa.bits)));
if (slowpath(sideTableLocked)) sidetable_unlock();
return false;
underflow:
newisa = oldisa;
if (slowpath(newisa.has_sidetable_rc)) {
if (!handleUnderflow) {
ClearExclusive(&isa.bits);
return rootRelease_underflow(performDealloc);
}
if (!sideTableLocked) {
ClearExclusive(&isa.bits);
sidetable_lock();
sideTableLocked = true;
goto retry;
}
size_t borrowed = sidetable_subExtraRC_nolock(RC_HALF);
if (borrowed > 0) {
newisa.extra_rc = borrowed - 1; bool stored = StoreReleaseExclusive(&isa.bits,
oldisa.bits, newisa.bits);
if (!stored) {
isa_t oldisa2 = LoadExclusive(&isa.bits);
isa_t newisa2 = oldisa2;
if (newisa2.nonpointer) {
uintptr_t overflow;
newisa2.bits =
addc(newisa2.bits, RC_ONE * (borrowed-1), 0, &overflow);
if (!overflow) {
stored = StoreReleaseExclusive(&isa.bits, oldisa2.bits,
newisa2.bits);
}
}
}
if (!stored) {
sidetable_addExtraRC_nolock(borrowed);
goto retry;
}
sidetable_unlock();
return false;
}
else {
}
}
if (slowpath(newisa.deallocating)) {
ClearExclusive(&isa.bits);
if (sideTableLocked) sidetable_unlock();
return overrelease_error();
}
newisa.deallocating = true;
if (!StoreExclusive(&isa.bits, oldisa.bits, newisa.bits)) goto retry;
if (slowpath(sideTableLocked)) sidetable_unlock();
__sync_synchronize();
if (performDealloc) {
((void(*)(objc_object *, SEL))objc_msgSend)(this, SEL_dealloc);
}
return true;
}
inline id
objc_object::autorelease()
{
if (isTaggedPointer()) return (id)this;
if (fastpath(!ISA()->hasCustomRR())) return rootAutorelease();
return ((id(*)(objc_object *, SEL))objc_msgSend)(this, SEL_autorelease);
}
inline id
objc_object::rootAutorelease()
{
if (isTaggedPointer()) return (id)this;
if (prepareOptimizedReturn(ReturnAtPlus1)) return (id)this;
return rootAutorelease2();
}
inline uintptr_t
objc_object::rootRetainCount()
{
if (isTaggedPointer()) return (uintptr_t)this;
sidetable_lock();
isa_t bits = LoadExclusive(&isa.bits);
ClearExclusive(&isa.bits);
if (bits.nonpointer) {
uintptr_t rc = 1 + bits.extra_rc;
if (bits.has_sidetable_rc) {
rc += sidetable_getExtraRC_nolock();
}
sidetable_unlock();
return rc;
}
sidetable_unlock();
return sidetable_retainCount();
}
#else
inline Class
objc_object::ISA()
{
assert(!isTaggedPointer());
return isa.cls;
}
inline bool
objc_object::hasNonpointerIsa()
{
return false;
}
inline void
objc_object::initIsa(Class cls)
{
assert(!isTaggedPointer());
isa = (uintptr_t)cls;
}
inline void
objc_object::initClassIsa(Class cls)
{
initIsa(cls);
}
inline void
objc_object::initProtocolIsa(Class cls)
{
initIsa(cls);
}
inline void
objc_object::initInstanceIsa(Class cls, bool)
{
initIsa(cls);
}
inline void
objc_object::initIsa(Class cls, bool, bool)
{
initIsa(cls);
}
inline Class
objc_object::changeIsa(Class cls)
{
assert(!isTaggedPointer());
isa_t oldisa, newisa;
newisa.cls = cls;
do {
oldisa = LoadExclusive(&isa.bits);
} while (!StoreExclusive(&isa.bits, oldisa.bits, newisa.bits));
if (oldisa.cls && oldisa.cls->instancesHaveAssociatedObjects()) {
cls->setInstancesHaveAssociatedObjects();
}
return oldisa.cls;
}
inline bool
objc_object::hasAssociatedObjects()
{
return getIsa()->instancesHaveAssociatedObjects();
}
inline void
objc_object::setHasAssociatedObjects()
{
getIsa()->setInstancesHaveAssociatedObjects();
}
inline bool
objc_object::isWeaklyReferenced()
{
assert(!isTaggedPointer());
return sidetable_isWeaklyReferenced();
}
inline void
objc_object::setWeaklyReferenced_nolock()
{
assert(!isTaggedPointer());
sidetable_setWeaklyReferenced_nolock();
}
inline bool
objc_object::hasCxxDtor()
{
assert(!isTaggedPointer());
return isa.cls->hasCxxDtor();
}
inline bool
objc_object::rootIsDeallocating()
{
if (isTaggedPointer()) return false;
return sidetable_isDeallocating();
}
inline void
objc_object::clearDeallocating()
{
sidetable_clearDeallocating();
}
inline void
objc_object::rootDealloc()
{
if (isTaggedPointer()) return;
object_dispose((id)this);
}
inline id
objc_object::retain()
{
assert(!isTaggedPointer());
if (fastpath(!ISA()->hasCustomRR())) {
return sidetable_retain();
}
return ((id(*)(objc_object *, SEL))objc_msgSend)(this, SEL_retain);
}
inline id
objc_object::rootRetain()
{
if (isTaggedPointer()) return (id)this;
return sidetable_retain();
}
inline void
objc_object::release()
{
assert(!isTaggedPointer());
if (fastpath(!ISA()->hasCustomRR())) {
sidetable_release();
return;
}
((void(*)(objc_object *, SEL))objc_msgSend)(this, SEL_release);
}
inline bool
objc_object::rootRelease()
{
if (isTaggedPointer()) return false;
return sidetable_release(true);
}
inline bool
objc_object::rootReleaseShouldDealloc()
{
if (isTaggedPointer()) return false;
return sidetable_release(false);
}
inline id
objc_object::autorelease()
{
if (isTaggedPointer()) return (id)this;
if (fastpath(!ISA()->hasCustomRR())) return rootAutorelease();
return ((id(*)(objc_object *, SEL))objc_msgSend)(this, SEL_autorelease);
}
inline id
objc_object::rootAutorelease()
{
if (isTaggedPointer()) return (id)this;
if (prepareOptimizedReturn(ReturnAtPlus1)) return (id)this;
return rootAutorelease2();
}
inline bool
objc_object::rootTryRetain()
{
if (isTaggedPointer()) return true;
return sidetable_tryRetain();
}
inline uintptr_t
objc_object::rootRetainCount()
{
if (isTaggedPointer()) return (uintptr_t)this;
return sidetable_retainCount();
}
#endif
#if SUPPORT_RETURN_AUTORELEASE
# if __x86_64__
static ALWAYS_INLINE bool
callerAcceptsOptimizedReturn(const void * const ra0)
{
const uint8_t *ra1 = (const uint8_t *)ra0;
const unaligned_uint16_t *ra2;
const unaligned_uint32_t *ra4 = (const unaligned_uint32_t *)ra1;
const void **sym;
#define PREFER_GOTPCREL 0
#if PREFER_GOTPCREL
if (*ra4 != 0xffc78948) {
return false;
}
if (ra1[4] != 0x15) {
return false;
}
ra1 += 3;
#else
if (*ra4 != 0xe8c78948) {
return false;
}
ra1 += (long)*(const unaligned_int32_t *)(ra1 + 4) + 8l;
ra2 = (const unaligned_uint16_t *)ra1;
if (*ra2 != 0x25ff) {
return false;
}
#endif
ra1 += 6l + (long)*(const unaligned_int32_t *)(ra1 + 2);
sym = (const void **)ra1;
if (*sym != objc_retainAutoreleasedReturnValue &&
*sym != objc_unsafeClaimAutoreleasedReturnValue)
{
return false;
}
return true;
}
# elif __arm__
static ALWAYS_INLINE bool
callerAcceptsOptimizedReturn(const void *ra)
{
if ((uintptr_t)ra & 1) {
if (*(uint16_t *)((uint8_t *)ra - 1) == 0x463f) {
return true;
}
} else {
if (*(unaligned_uint32_t *)ra == 0xe1a07007) {
return true;
}
}
return false;
}
# elif __arm64__
static ALWAYS_INLINE bool
callerAcceptsOptimizedReturn(const void *ra)
{
if (*(uint32_t *)ra == 0xaa1d03fd) {
return true;
}
return false;
}
# elif __i386__
static ALWAYS_INLINE bool
callerAcceptsOptimizedReturn(const void *ra)
{
if (*(unaligned_uint16_t *)ra == 0xed89) {
return true;
}
return false;
}
# else
#warning unknown architecture
static ALWAYS_INLINE bool
callerAcceptsOptimizedReturn(const void *ra)
{
return false;
}
# endif
static ALWAYS_INLINE ReturnDisposition
getReturnDisposition()
{
return (ReturnDisposition)(uintptr_t)tls_get_direct(RETURN_DISPOSITION_KEY);
}
static ALWAYS_INLINE void
setReturnDisposition(ReturnDisposition disposition)
{
tls_set_direct(RETURN_DISPOSITION_KEY, (void*)(uintptr_t)disposition);
}
static ALWAYS_INLINE bool
prepareOptimizedReturn(ReturnDisposition disposition)
{
assert(getReturnDisposition() == ReturnAtPlus0);
if (callerAcceptsOptimizedReturn(__builtin_return_address(0))) {
if (disposition) setReturnDisposition(disposition);
return true;
}
return false;
}
static ALWAYS_INLINE ReturnDisposition
acceptOptimizedReturn()
{
ReturnDisposition disposition = getReturnDisposition();
setReturnDisposition(ReturnAtPlus0); return disposition;
}
#else
static ALWAYS_INLINE bool
prepareOptimizedReturn(ReturnDisposition disposition __unused)
{
return false;
}
static ALWAYS_INLINE ReturnDisposition
acceptOptimizedReturn()
{
return ReturnAtPlus0;
}
#endif
#endif