#include <mach/mach_types.h>
#include <mach/machine.h>
#include <mach/vm_map.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
#include <machine/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/vm_protos.h>
#include <ipc/ipc_port.h>
#include <arm/cpuid.h>
#include <arm/rtclock.h>
#include <libkern/OSAtomic.h>
#include <stdatomic.h>
#include <sys/kdebug.h>
#if CONFIG_ATM
#include <atm/atm_internal.h>
#endif
static void commpage_init_cpu_capabilities( void );
static int commpage_cpus( void );
vm_address_t commPagePtr=0;
vm_address_t sharedpage_rw_addr = 0;
uint32_t _cpu_capabilities = 0;
extern int gARMv81Atomics;
void
commpage_populate(
void)
{
uint16_t c2;
int cpufamily;
sharedpage_rw_addr = pmap_create_sharedpage();
commPagePtr = (vm_address_t)_COMM_PAGE_BASE_ADDRESS;
*((uint16_t*)(_COMM_PAGE_VERSION+_COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION;
commpage_init_cpu_capabilities();
commpage_set_timestamp(0, 0, 0, 0, 0);
if (_cpu_capabilities & kCache32)
c2 = 32;
else if (_cpu_capabilities & kCache64)
c2 = 64;
else if (_cpu_capabilities & kCache128)
c2 = 128;
else
c2 = 0;
*((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE+_COMM_PAGE_RW_OFFSET)) = c2;
*((uint32_t*)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = 1;
commpage_update_active_cpus();
cpufamily = cpuid_get_cpufamily();
*((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS+_COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max;
*((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS+_COMM_PAGE_RW_OFFSET))= (uint8_t) machine_info.logical_cpu_max;
*((uint64_t*)(_COMM_PAGE_MEMORY_SIZE+_COMM_PAGE_RW_OFFSET)) = machine_info.max_mem;
*((uint32_t*)(_COMM_PAGE_CPUFAMILY+_COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily;
*((uint32_t*)(_COMM_PAGE_DEV_FIRM+_COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
*((uint8_t*)(_COMM_PAGE_USER_TIMEBASE+_COMM_PAGE_RW_OFFSET)) = user_timebase_allowed();
*((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK+_COMM_PAGE_RW_OFFSET)) = user_cont_hwclock_allowed();
*((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift;
#if __arm64__
*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32;
*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS)
*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
#else
*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
*((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
#endif
commpage_update_timebase();
commpage_update_mach_continuous_time(0);
clock_sec_t secs;
clock_usec_t microsecs;
clock_get_boottime_microtime(&secs, µsecs);
commpage_update_boottime(secs * USEC_PER_SEC + microsecs);
*((uint64_t *)(_COMM_PAGE_APPROX_TIME+ _COMM_PAGE_RW_OFFSET)) = 0;
#ifdef CONFIG_MACH_APPROXIMATE_TIME
*((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 1;
#else
*((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 0;
#endif
commpage_update_kdebug_state();
#if CONFIG_ATM
commpage_update_atm_diagnostic_config(atm_get_diagnostic_config());
#endif
}
struct mu {
uint64_t m; int32_t a; int32_t s; };
void
commpage_set_timestamp(
uint64_t tbr,
uint64_t secs,
uint64_t frac,
uint64_t scale,
uint64_t tick_per_sec)
{
new_commpage_timeofday_data_t *commpage_timeofday_datap;
if (commPagePtr == 0)
return;
commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA+_COMM_PAGE_RW_OFFSET);
commpage_timeofday_datap->TimeStamp_tick = 0x0ULL;
#if (__ARM_ARCH__ >= 7)
__asm__ volatile("dmb ish");
#endif
commpage_timeofday_datap->TimeStamp_sec = secs;
commpage_timeofday_datap->TimeStamp_frac = frac;
commpage_timeofday_datap->Ticks_scale = scale;
commpage_timeofday_datap->Ticks_per_sec = tick_per_sec;
#if (__ARM_ARCH__ >= 7)
__asm__ volatile("dmb ish");
#endif
commpage_timeofday_datap->TimeStamp_tick = tbr;
}
void
commpage_set_memory_pressure(
unsigned int pressure )
{
if (commPagePtr == 0)
return;
*((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE+_COMM_PAGE_RW_OFFSET)) = pressure;
}
void
commpage_set_spin_count(
unsigned int count )
{
if (count == 0)
count = 1;
if (commPagePtr == 0)
return;
*((uint32_t *)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = count;
}
static int
commpage_cpus( void )
{
int cpus;
cpus = ml_get_max_cpus();
if (cpus == 0)
panic("commpage cpus==0");
if (cpus > 0xFF)
cpus = 0xFF;
return cpus;
}
static void
commpage_init_cpu_capabilities( void )
{
uint32_t bits;
int cpus;
ml_cpu_info_t cpu_info;
bits = 0;
ml_cpu_get_info(&cpu_info);
switch (cpu_info.cache_line_size) {
case 128:
bits |= kCache128;
break;
case 64:
bits |= kCache64;
break;
case 32:
bits |= kCache32;
break;
default:
break;
}
cpus = commpage_cpus();
if (cpus == 1)
bits |= kUP;
bits |= (cpus << kNumCPUsShift);
bits |= kFastThreadLocalStorage; #if __ARM_VFP__
bits |= kHasVfp;
#if (__ARM_VFP__ >= 3)
bits |= kHasNeon;
#if defined(__arm64__)
bits |= kHasNeonHPFP;
#else
boolean_t intr = ml_set_interrupts_enabled(FALSE);
unsigned int mvfr1 = get_mvfr1();
if (mvfr1 & MVFR_ASIMD_HPFP)
bits |= kHasNeonHPFP;
(void) ml_set_interrupts_enabled(intr);
#endif
#endif
#endif
#if defined(__arm64__)
bits |= kHasFMA;
#endif
#if __ARM_ENABLE_WFE_
#ifdef __arm64__
if (arm64_wfe_allowed()) {
bits |= kHasEvent;
}
#else
bits |= kHasEvent;
#endif
#endif
#if __ARM_V8_CRYPTO_EXTENSIONS__
bits |= kHasARMv8Crypto;
#endif
#ifdef __arm64__
if ((__builtin_arm_rsr64("ID_AA64ISAR0_EL1") & ID_AA64ISAR0_EL1_ATOMIC_MASK) == ID_AA64ISAR0_EL1_ATOMIC_8_1) {
bits |= kHasARMv81Atomics;
gARMv81Atomics = 1;
}
#endif
_cpu_capabilities = bits;
*((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES+_COMM_PAGE_RW_OFFSET)) = _cpu_capabilities;
}
void
commpage_update_active_cpus(void)
{
if (!commPagePtr)
return;
*((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS+_COMM_PAGE_RW_OFFSET)) = processor_avail_count;
}
void
commpage_update_timebase(void)
{
if (commPagePtr) {
*((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET+_COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime;
}
}
void
commpage_update_kdebug_state(void)
{
if (commPagePtr)
*((volatile uint32_t*)(_COMM_PAGE_KDEBUG_ENABLE+_COMM_PAGE_RW_OFFSET)) = kdebug_commpage_state();
}
void
commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)
{
if (commPagePtr)
*((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG+_COMM_PAGE_RW_OFFSET)) = diagnostic_config;
}
void
commpage_update_multiuser_config(uint32_t multiuser_config)
{
if (commPagePtr)
*((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG+_COMM_PAGE_RW_OFFSET)) = multiuser_config;
}
void
commpage_update_mach_approximate_time(uint64_t abstime)
{
#ifdef CONFIG_MACH_APPROXIMATE_TIME
uintptr_t approx_time_base = (uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET);
uint64_t saved_data;
if (commPagePtr) {
saved_data = atomic_load_explicit((_Atomic uint64_t *)approx_time_base,
memory_order_relaxed);
if (saved_data < abstime) {
atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)approx_time_base,
&saved_data, abstime, memory_order_relaxed, memory_order_relaxed);
}
}
#else
#pragma unused (abstime)
#endif
}
void
commpage_update_mach_continuous_time(uint64_t sleeptime)
{
if (commPagePtr) {
#ifdef __arm64__
*((uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = sleeptime;
#else
uint64_t *c_time_base = (uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET);
uint64_t old;
do {
old = *c_time_base;
} while(!OSCompareAndSwap64(old, sleeptime, c_time_base));
#endif
}
}
void
commpage_update_boottime(uint64_t value)
{
if (commPagePtr) {
#ifdef __arm64__
*((uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET)) = value;
#else
uint64_t *cp = (uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET);
uint64_t old_value;
do {
old_value = *cp;
} while (!OSCompareAndSwap64(old_value, value, cp));
#endif
}
}