locks.h   [plain text]

/*
 * Copyright (c) 2007-2017 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef	_ARM_LOCKS_H_
#define	_ARM_LOCKS_H_

#include <kern/kern_types.h>
#ifdef	MACH_KERNEL_PRIVATE
#include <arm/hw_lock_types.h>
#endif


#ifdef	MACH_KERNEL_PRIVATE

extern	unsigned int	LcksOpts;

#define enaLkDeb		0x00000001	/* Request debug in default attribute */
#define enaLkStat		0x00000002	/* Request statistic in default attribute */
#define disLkRWPrio		0x00000004	/* Disable RW lock priority promotion */

#define disLkType		0x80000000	/* Disable type checking */
#define disLktypeb		0
#define disLkThread		0x40000000	/* Disable ownership checking */
#define disLkThreadb	1
#define enaLkExtStck	0x20000000	/* Enable extended backtrace */
#define enaLkExtStckb	2
#define disLkMyLck		0x10000000	/* Disable recursive lock dectection */
#define disLkMyLckb		3

#endif

#ifdef	MACH_KERNEL_PRIVATE
typedef struct {
	struct hslock	hwlock;
	uintptr_t		type;
} lck_spin_t;

#define lck_spin_data hwlock.lock_data

#define	LCK_SPIN_TAG_DESTROYED	0xdead	/* lock marked as Destroyed */

#define	LCK_SPIN_TYPE			0x00000011

#else
#ifdef	KERNEL_PRIVATE

typedef struct {
	uintptr_t   		opaque[2];
} lck_spin_t;

#else
typedef struct __lck_spin_t__	lck_spin_t;
#endif	// KERNEL_PRIVATE
#endif	// MACH_KERNEL_PRIVATE

#ifdef	MACH_KERNEL_PRIVATE
typedef struct _lck_mtx_ {
	union {

		uintptr_t					lck_mtx_data;	/* Thread pointer plus lock bits */
		uintptr_t					lck_mtx_tag;	/* Tag for type */
	};												/* arm: 4   arm64: 8 */
	union {
		struct {
			uint16_t				lck_mtx_waiters;/* Number of waiters */
			uint8_t					lck_mtx_pri;	/* Priority to inherit */
			uint8_t					lck_mtx_type;	/* Type */
		};
		struct {
			struct _lck_mtx_ext_	*lck_mtx_ptr;	/* Indirect pointer */
		};
	};												/* arm: 4   arm64: 8 */
} lck_mtx_t;										/* arm: 8  arm64: 16 */

/* Shared between mutex and read-write locks */
#define LCK_ILOCK_BIT		0
#define ARM_LCK_WAITERS_BIT	1
#define LCK_ILOCK			(1 << LCK_ILOCK_BIT)
#define ARM_LCK_WAITERS		(1 << ARM_LCK_WAITERS_BIT)

#define	LCK_MTX_TYPE					0x22		/* lock type */

#define	LCK_MTX_TAG_INDIRECT			0x00001007	/* lock marked as Indirect  */
#define	LCK_MTX_TAG_DESTROYED			0x00002007	/* lock marked as Destroyed */

#define	LCK_FRAMES_MAX	8

extern uint64_t 	MutexSpin;

typedef struct {
	unsigned int		type;
	vm_offset_t			stack[LCK_FRAMES_MAX];
	vm_offset_t			thread;
} lck_mtx_deb_t;

#define MUTEX_TAG       0x4d4d

typedef struct {
	unsigned int		lck_mtx_stat_data;
} lck_mtx_stat_t;

typedef struct _lck_mtx_ext_ {
	lck_mtx_t			lck_mtx;        /* arm: 12  arm64: 24 */
	struct _lck_grp_	*lck_mtx_grp;   /* arm: 4   arm64: 8 */
	unsigned int		lck_mtx_attr;   /* arm: 4   arm64: 4 */
	lck_mtx_stat_t		lck_mtx_stat;   /* arm: 4   arm64: 4 */
	lck_mtx_deb_t		lck_mtx_deb;    /* arm: 40  arm64: 80 */
} lck_mtx_ext_t;                        /* arm: 64  arm64: 120 */

#define	LCK_MTX_ATTR_DEBUG	0x1
#define	LCK_MTX_ATTR_DEBUGb	31
#define	LCK_MTX_ATTR_STAT	0x2
#define	LCK_MTX_ATTR_STATb	30

#define LCK_MTX_EVENT(lck)        ((event_t)(((unsigned int*)(lck))+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))

#else
#ifdef	KERNEL_PRIVATE
typedef struct {
    uintptr_t        opaque[2];
} lck_mtx_t;

typedef struct {
#if defined(__arm64__)
    unsigned long       opaque[16];
#else /* __arm__ */
	unsigned int		opaque[16];
#endif 
} lck_mtx_ext_t;

#else
typedef struct __lck_mtx_t__	lck_mtx_t;
#endif
#endif

#ifdef	MACH_KERNEL_PRIVATE

typedef union {
	struct {
		uint16_t	shared_count;		/* No. of shared granted request */
		uint16_t	interlock:		1,	/* Interlock */
					priv_excl:		1,	/* priority for Writer */
					want_upgrade:	1,	/* Read-to-write upgrade waiting */
					want_excl:		1,	/* Writer is waiting, or locked for write */
					r_waiting:		1,	/* Someone is sleeping on lock */
					w_waiting:		1,	/* Writer is sleeping on lock */
					can_sleep:		1,	/* Can attempts to lock go to sleep? */
					_pad2:			8,	/* padding */
					tag_valid:		1;	/* Field is actually a tag, not a bitfield */
#if __arm64__
		uint32_t	_pad4;
#endif
	};
	struct {
		uint32_t	data;						/* Single word version of bitfields and shared count */
#if __arm64__
		uint32_t 	lck_rw_pad4;
#endif
	};
} lck_rw_word_t;

typedef struct {
	lck_rw_word_t	word;
	thread_t		lck_rw_owner;
} lck_rw_t;	/* arm: 8  arm64: 16 */

#define lck_rw_shared_count	word.shared_count
#define lck_rw_interlock	word.interlock
#define lck_rw_priv_excl	word.priv_excl
#define lck_rw_want_upgrade	word.want_upgrade
#define lck_rw_want_excl	word.want_excl
#define lck_r_waiting		word.r_waiting
#define lck_w_waiting		word.w_waiting
#define lck_rw_can_sleep	word.can_sleep
#define lck_rw_data			word.data
// tag and data reference the same memory. When the tag_valid bit is set,
// the data word should be treated as a tag instead of a bitfield.
#define lck_rw_tag_valid	word.tag_valid
#define lck_rw_tag			word.data

#define LCK_RW_SHARED_READER_OFFSET	 0
#define LCK_RW_INTERLOCK_BIT		16
#define LCK_RW_PRIV_EXCL_BIT		17
#define LCK_RW_WANT_UPGRADE_BIT		18
#define LCK_RW_WANT_EXCL_BIT		19
#define LCK_RW_R_WAITING_BIT		20
#define LCK_RW_W_WAITING_BIT		21
#define LCK_RW_CAN_SLEEP_BIT		22
//									23-30
#define LCK_RW_TAG_VALID_BIT		31

#define LCK_RW_INTERLOCK		(1 << LCK_RW_INTERLOCK_BIT)
#define LCK_RW_R_WAITING		(1 << LCK_RW_R_WAITING_BIT)
#define LCK_RW_W_WAITING		(1 << LCK_RW_W_WAITING_BIT)
#define LCK_RW_WANT_UPGRADE		(1 << LCK_RW_WANT_UPGRADE_BIT)
#define LCK_RW_WANT_EXCL		(1 << LCK_RW_WANT_EXCL_BIT)
#define LCK_RW_TAG_VALID		(1 << LCK_RW_TAG_VALID_BIT)
#define LCK_RW_PRIV_EXCL		(1 << LCK_RW_PRIV_EXCL_BIT)
#define LCK_RW_SHARED_MASK		(0xffff << LCK_RW_SHARED_READER_OFFSET)
#define LCK_RW_SHARED_READER	(0x1 << LCK_RW_SHARED_READER_OFFSET)

#define	LCK_RW_TAG_DESTROYED		((LCK_RW_TAG_VALID | 0xdddddeadu))	/* lock marked as Destroyed */

#define LCK_RW_WRITER_EVENT(lck)		(event_t)((uintptr_t)(lck)+1)
#define LCK_RW_READER_EVENT(lck)		(event_t)((uintptr_t)(lck)+2)
#define WRITE_EVENT_TO_RWLOCK(event)	((lck_rw_t *)((uintptr_t)(event)-1))
#define READ_EVENT_TO_RWLOCK(event)		((lck_rw_t *)((uintptr_t)(event)-2))

#if __ARM_ENABLE_WFE_

#define wait_for_event()	__builtin_arm_wfe()
#if __arm__
#define set_event()			do{__builtin_arm_dsb(DSB_ISHST);__builtin_arm_sev();}while(0)
#define LOCK_SNOOP_SPINS	4
#else
#define set_event()			do{}while(0)	// arm64 sev is implicit in stlxr
#define LOCK_SNOOP_SPINS	0x300
#endif

#else

#define wait_for_event()	__builtin_arm_clrex()
#define set_event()		do{}while(0)
#define LOCK_SNOOP_SPINS	0x300

#endif // __ARM_ENABLE_WFE_

#if LOCK_PRIVATE

#define LOCK_PANIC_TIMEOUT	0xc00000	// 12.5 m ticks = 250ms with 24MHz OSC

#define LOCK_TRY_DISABLE_INT 1	// Disable interrupts for a quick acquire attempt

#define PLATFORM_LCK_ILOCK LCK_ILOCK


/*
 * Lock state to thread pointer
 * Clear the bottom bits
 */
#define LCK_MTX_STATE_TO_THREAD(s)	(thread_t)(s & ~(LCK_ILOCK | ARM_LCK_WAITERS))
/*
 * Thread pointer to lock state
 * arm thread pointers are aligned such that the bottom two bits are clear
 */
#define LCK_MTX_THREAD_TO_STATE(t) 	((uintptr_t)t)
/*
 * Thread pointer mask
 */
#define LCK_MTX_THREAD_MASK (~(uintptr_t)(LCK_ILOCK | ARM_LCK_WAITERS))

#define disable_preemption_for_thread(t) ((volatile thread_t)t)->machine.preemption_count++


__unused static void disable_interrupts_noread(void)
{
#if __arm__
	__asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ
#else
	__builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));	// Mask IRQ FIQ
#endif
}

__unused static inline long get_interrupts(void)
{
	long	state;

#if __arm__
	__asm__ volatile ("mrs %[state], cpsr" :[state] "=r" (state));	// Read cpsr
#else
	state = __builtin_arm_rsr64("DAIF");	// Read interrupt state
#endif
	return state;
}

__unused static inline long disable_interrupts(void)
{
	long	state;
	
	state = get_interrupts();		// Get previous state
	disable_interrupts_noread();	// Disable
	return state;
}

__unused static inline void restore_interrupts(long state)
{
#if __arm__
	__asm__ volatile ("msr  cpsr, %[state]" :: [state] "r" (state) : "cc", "memory"); // Restore CPSR
#elif __arm64__
	__builtin_arm_wsr64("DAIF", state);	// Restore masks
#endif
}

#endif // LOCK_PRIVATE

#else
#ifdef	KERNEL_PRIVATE
typedef struct {
    uintptr_t   		 opaque[2];
} lck_rw_t;
#else
typedef	struct __lck_rw_t__	lck_rw_t;
#endif
#endif

#endif	/* _ARM_LOCKS_H_ */