machine_routines.c [plain text]
#include <arm64/proc_reg.h>
#include <arm/machine_cpu.h>
#include <arm/cpu_internal.h>
#include <arm/cpuid.h>
#include <arm/io_map_entries.h>
#include <arm/cpu_data.h>
#include <arm/cpu_data_internal.h>
#include <arm/caches_internal.h>
#include <arm/misc_protos.h>
#include <arm/machdep_call.h>
#include <arm/rtclock.h>
#include <console/serial_protos.h>
#include <kern/machine.h>
#include <prng/random.h>
#include <kern/startup.h>
#include <kern/thread.h>
#include <mach/machine.h>
#include <machine/atomic.h>
#include <vm/pmap.h>
#include <vm/vm_page.h>
#include <sys/kdebug.h>
#include <kern/coalition.h>
#include <pexpert/device_tree.h>
#include <IOKit/IOPlatformExpert.h>
#if defined(KERNEL_INTEGRITY_KTRR)
#include <libkern/kernel_mach_header.h>
#endif
#include <libkern/section_keywords.h>
#if KPC
#include <kern/kpc.h>
#endif
static int max_cpus_initialized = 0;
#define MAX_CPUS_SET 0x1
#define MAX_CPUS_WAIT 0x2
uint32_t LockTimeOut;
uint32_t LockTimeOutUsec;
uint64_t MutexSpin;
boolean_t is_clock_configured = FALSE;
uint32_t yield_delay_us = 42;
extern int mach_assert;
extern volatile uint32_t debug_enabled;
extern vm_offset_t segEXTRADATA;
extern vm_offset_t segLOWESTTEXT;
extern vm_offset_t segLASTB;
extern unsigned long segSizeLAST;
void machine_conf(void);
thread_t Idle_context(void);
SECURITY_READ_ONLY_LATE(static uint32_t) cpu_phys_ids[MAX_CPUS] = {[0 ... MAX_CPUS - 1] = (uint32_t)-1};
SECURITY_READ_ONLY_LATE(static unsigned int) avail_cpus = 0;
SECURITY_READ_ONLY_LATE(static int) boot_cpu = -1;
SECURITY_READ_ONLY_LATE(static int) max_cpu_number = 0;
SECURITY_READ_ONLY_LATE(cluster_type_t) boot_cluster = CLUSTER_TYPE_SMP;
SECURITY_READ_ONLY_LATE(static uint32_t) fiq_eventi = UINT32_MAX;
lockdown_handler_t lockdown_handler;
void *lockdown_this;
lck_mtx_t lockdown_handler_lck;
lck_grp_t *lockdown_handler_grp;
int lockdown_done;
void ml_lockdown_init(void);
void ml_lockdown_run_handler(void);
uint32_t get_arm_cpu_version(void);
void ml_cpu_signal(unsigned int cpu_id __unused)
{
panic("Platform does not support ACC Fast IPI");
}
void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs) {
(void)nanosecs;
panic("Platform does not support ACC Fast IPI");
}
uint64_t ml_cpu_signal_deferred_get_timer() {
return 0;
}
void ml_cpu_signal_deferred(unsigned int cpu_id __unused)
{
panic("Platform does not support ACC Fast IPI deferral");
}
void ml_cpu_signal_retract(unsigned int cpu_id __unused)
{
panic("Platform does not support ACC Fast IPI retraction");
}
void machine_idle(void)
{
__asm__ volatile ("msr DAIFSet, %[mask]" ::[mask] "i" (DAIFSC_IRQF | DAIFSC_FIQF));
Idle_context();
__asm__ volatile ("msr DAIFClr, %[mask]" ::[mask] "i" (DAIFSC_IRQF | DAIFSC_FIQF));
}
void init_vfp(void)
{
return;
}
boolean_t get_vfp_enabled(void)
{
return TRUE;
}
void OSSynchronizeIO(void)
{
__builtin_arm_dsb(DSB_SY);
}
uint64_t get_aux_control(void)
{
uint64_t value;
MRS(value, "ACTLR_EL1");
return value;
}
uint64_t get_mmu_control(void)
{
uint64_t value;
MRS(value, "SCTLR_EL1");
return value;
}
uint64_t get_tcr(void)
{
uint64_t value;
MRS(value, "TCR_EL1");
return value;
}
boolean_t ml_get_interrupts_enabled(void)
{
uint64_t value;
MRS(value, "DAIF");
if (value & DAIF_IRQF)
return FALSE;
return TRUE;
}
pmap_paddr_t get_mmu_ttb(void)
{
pmap_paddr_t value;
MRS(value, "TTBR0_EL1");
return value;
}
static uint32_t get_midr_el1(void)
{
uint64_t value;
MRS(value, "MIDR_EL1");
return (uint32_t) value;
}
uint32_t get_arm_cpu_version(void)
{
uint32_t value = get_midr_el1();
return ((value & MIDR_EL1_REV_MASK) >> MIDR_EL1_REV_SHIFT) | ((value & MIDR_EL1_VAR_MASK) >> (MIDR_EL1_VAR_SHIFT - 4));
}
boolean_t user_cont_hwclock_allowed(void)
{
return FALSE;
}
boolean_t user_timebase_allowed(void)
{
return TRUE;
}
boolean_t arm64_wfe_allowed(void)
{
return TRUE;
}
#if defined(KERNEL_INTEGRITY_KTRR)
uint64_t rorgn_begin __attribute__((section("__DATA, __const"))) = 0;
uint64_t rorgn_end __attribute__((section("__DATA, __const"))) = 0;
vm_offset_t amcc_base;
static void assert_unlocked(void);
static void assert_amcc_cache_disabled(void);
static void lock_amcc(void);
static void lock_mmu(uint64_t begin, uint64_t end);
void rorgn_stash_range(void)
{
#if DEVELOPMENT || DEBUG
boolean_t rorgn_disable = FALSE;
PE_parse_boot_argn("-unsafe_kernel_text", &rorgn_disable, sizeof(rorgn_disable));
if (rorgn_disable) {
return;
}
#endif
uint64_t dram_base = gPhysBase & ~0x1FFFFFFFULL;
assert(dram_base == 0x800000000ULL);
#if defined(KERNEL_INTEGRITY_KTRR)
uint64_t soc_base = 0;
DTEntry entryP = NULL;
uintptr_t *reg_prop = NULL;
uint32_t prop_size = 0;
int rc;
soc_base = pe_arm_get_soc_base_phys();
rc = DTFindEntry("name", "mcc", &entryP);
assert(rc == kSuccess);
rc = DTGetProperty(entryP, "reg", (void **)®_prop, &prop_size);
assert(rc == kSuccess);
amcc_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
#else
#error "KERNEL_INTEGRITY config error"
#endif
#if defined(KERNEL_INTEGRITY_KTRR)
assert(rRORGNENDADDR > rRORGNBASEADDR);
rorgn_begin = (rRORGNBASEADDR << AMCC_PGSHIFT) + dram_base;
rorgn_end = (rRORGNENDADDR << AMCC_PGSHIFT) + dram_base;
#else
#error KERNEL_INTEGRITY config error
#endif
}
static void assert_unlocked() {
uint64_t ktrr_lock = 0;
uint32_t rorgn_lock = 0;
assert(amcc_base);
#if defined(KERNEL_INTEGRITY_KTRR)
rorgn_lock = rRORGNLOCK;
ktrr_lock = __builtin_arm_rsr64(ARM64_REG_KTRR_LOCK_EL1);
#else
#error KERNEL_INTEGRITY config error
#endif
assert(!ktrr_lock);
assert(!rorgn_lock);
}
static void lock_amcc() {
#if defined(KERNEL_INTEGRITY_KTRR)
rRORGNLOCK = 1;
__builtin_arm_isb(ISB_SY);
#else
#error KERNEL_INTEGRITY config error
#endif
}
static void lock_mmu(uint64_t begin, uint64_t end) {
#if defined(KERNEL_INTEGRITY_KTRR)
__builtin_arm_wsr64(ARM64_REG_KTRR_LOWER_EL1, begin);
__builtin_arm_wsr64(ARM64_REG_KTRR_UPPER_EL1, end);
__builtin_arm_wsr64(ARM64_REG_KTRR_LOCK_EL1, 1ULL);
__builtin_arm_isb(ISB_SY);
flush_mmu_tlb();
#else
#error KERNEL_INTEGRITY config error
#endif
}
static void assert_amcc_cache_disabled() {
#if defined(KERNEL_INTEGRITY_KTRR)
assert((rMCCGEN & 1) == 0);
#else
#error KERNEL_INTEGRITY config error
#endif
}
void rorgn_lockdown(void)
{
vm_offset_t ktrr_begin, ktrr_end;
unsigned long last_segsz;
#if DEVELOPMENT || DEBUG
boolean_t ktrr_disable = FALSE;
PE_parse_boot_argn("-unsafe_kernel_text", &ktrr_disable, sizeof(ktrr_disable));
if (ktrr_disable) {
goto out;
}
#endif
assert_unlocked();
ktrr_begin = segEXTRADATA;
assert(ktrr_begin && gVirtBase && gPhysBase);
ktrr_begin = kvtophys(ktrr_begin);
ktrr_end = kvtophys(segLASTB);
last_segsz = segSizeLAST;
#if defined(KERNEL_INTEGRITY_KTRR)
ktrr_end = (ktrr_end - 1) & ~AMCC_PGMASK;
assert(rorgn_begin == ktrr_begin && rorgn_end == (ktrr_end + last_segsz));
assert(last_segsz == PAGE_SIZE);
#endif
#if DEBUG || DEVELOPMENT
printf("KTRR Begin: %p End: %p, setting lockdown\n", (void *)ktrr_begin, (void *)ktrr_end);
#endif
assert_amcc_cache_disabled();
CleanPoC_DcacheRegion_Force(phystokv(ktrr_begin),
(unsigned)((ktrr_end + last_segsz) - ktrr_begin + AMCC_PGMASK));
lock_amcc();
lock_mmu(ktrr_begin, ktrr_end);
#if DEVELOPMENT || DEBUG
out:
#endif
ml_lockdown_run_handler();
}
#endif
void
machine_startup(__unused boot_args * args)
{
int boot_arg;
PE_parse_boot_argn("assert", &mach_assert, sizeof (mach_assert));
if (PE_parse_boot_argn("preempt", &boot_arg, sizeof (boot_arg))) {
default_preemption_rate = boot_arg;
}
if (PE_parse_boot_argn("bg_preempt", &boot_arg, sizeof (boot_arg))) {
default_bg_preemption_rate = boot_arg;
}
PE_parse_boot_argn("yield_delay_us", &yield_delay_us, sizeof (yield_delay_us));
machine_conf();
kernel_bootstrap();
}
void machine_lockdown_preflight(void)
{
#if CONFIG_KERNEL_INTEGRITY
#if defined(KERNEL_INTEGRITY_KTRR)
rorgn_stash_range();
#endif
#endif
}
void machine_lockdown(void)
{
#if CONFIG_KERNEL_INTEGRITY
#if KERNEL_INTEGRITY_WT
#ifdef MONITOR
monitor_call(MONITOR_LOCKDOWN, 0, 0, 0);
#endif
#endif
#if defined(KERNEL_INTEGRITY_KTRR)
rorgn_lockdown();
#endif
#endif
}
char *
machine_boot_info(
__unused char *buf,
__unused vm_size_t size)
{
return (PE_boot_args());
}
void
machine_conf(void)
{
machine_info.memory_size = (uint32_t)mem_size;
}
void
machine_init(void)
{
debug_log_init();
clock_config();
is_clock_configured = TRUE;
if (debug_enabled)
pmap_map_globals();
}
void
slave_machine_init(__unused void *param)
{
cpu_machine_init();
clock_init();
}
thread_t
machine_processor_shutdown(
__unused thread_t thread,
void (*doshutdown) (processor_t),
processor_t processor)
{
return (Shutdown_context(doshutdown, processor));
}
void
ml_init_max_cpus(unsigned int max_cpus)
{
boolean_t current_state;
current_state = ml_set_interrupts_enabled(FALSE);
if (max_cpus_initialized != MAX_CPUS_SET) {
machine_info.max_cpus = max_cpus;
machine_info.physical_cpu_max = max_cpus;
machine_info.logical_cpu_max = max_cpus;
if (max_cpus_initialized == MAX_CPUS_WAIT)
thread_wakeup((event_t) & max_cpus_initialized);
max_cpus_initialized = MAX_CPUS_SET;
}
(void) ml_set_interrupts_enabled(current_state);
}
unsigned int
ml_get_max_cpus(void)
{
boolean_t current_state;
current_state = ml_set_interrupts_enabled(FALSE);
if (max_cpus_initialized != MAX_CPUS_SET) {
max_cpus_initialized = MAX_CPUS_WAIT;
assert_wait((event_t) & max_cpus_initialized, THREAD_UNINT);
(void) thread_block(THREAD_CONTINUE_NULL);
}
(void) ml_set_interrupts_enabled(current_state);
return (machine_info.max_cpus);
}
void
ml_init_lock_timeout(void)
{
uint64_t abstime;
uint64_t mtxspin;
uint64_t default_timeout_ns = NSEC_PER_SEC>>2;
uint32_t slto;
if (PE_parse_boot_argn("slto_us", &slto, sizeof (slto)))
default_timeout_ns = slto * NSEC_PER_USEC;
nanoseconds_to_absolutetime(default_timeout_ns, &abstime);
LockTimeOutUsec = (uint32_t)(abstime / NSEC_PER_USEC);
LockTimeOut = (uint32_t)abstime;
if (PE_parse_boot_argn("mtxspin", &mtxspin, sizeof (mtxspin))) {
if (mtxspin > USEC_PER_SEC>>4)
mtxspin = USEC_PER_SEC>>4;
nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime);
} else {
nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime);
}
MutexSpin = abstime;
}
void
ml_cpu_up(void)
{
hw_atomic_add(&machine_info.physical_cpu, 1);
hw_atomic_add(&machine_info.logical_cpu, 1);
}
void
ml_cpu_down(void)
{
cpu_data_t *cpu_data_ptr;
hw_atomic_sub(&machine_info.physical_cpu, 1);
hw_atomic_sub(&machine_info.logical_cpu, 1);
cpu_data_ptr = getCpuDatap();
cpu_data_ptr->cpu_running = FALSE;
cpu_signal_handler_internal(TRUE);
}
void
ml_cpu_get_info(ml_cpu_info_t * ml_cpu_info)
{
cache_info_t *cpuid_cache_info;
cpuid_cache_info = cache_info();
ml_cpu_info->vector_unit = 0;
ml_cpu_info->cache_line_size = cpuid_cache_info->c_linesz;
ml_cpu_info->l1_icache_size = cpuid_cache_info->c_isize;
ml_cpu_info->l1_dcache_size = cpuid_cache_info->c_dsize;
#if (__ARM_ARCH__ >= 7)
ml_cpu_info->l2_settings = 1;
ml_cpu_info->l2_cache_size = cpuid_cache_info->c_l2size;
#else
ml_cpu_info->l2_settings = 0;
ml_cpu_info->l2_cache_size = 0xFFFFFFFF;
#endif
ml_cpu_info->l3_settings = 0;
ml_cpu_info->l3_cache_size = 0xFFFFFFFF;
}
unsigned int
ml_get_machine_mem(void)
{
return (machine_info.memory_size);
}
__attribute__((noreturn))
void
halt_all_cpus(boolean_t reboot)
{
if (reboot) {
printf("MACH Reboot\n");
PEHaltRestart(kPERestartCPU);
} else {
printf("CPU halted\n");
PEHaltRestart(kPEHaltCPU);
}
while (1);
}
__attribute__((noreturn))
void
halt_cpu(void)
{
halt_all_cpus(FALSE);
}
void
machine_signal_idle(
processor_t processor)
{
cpu_signal(processor_to_cpu_datap(processor), SIGPnop, (void *)NULL, (void *)NULL);
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 0 , 0, 0, 0);
}
void
machine_signal_idle_deferred(
processor_t processor)
{
cpu_signal_deferred(processor_to_cpu_datap(processor));
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_DEFERRED_AST), processor->cpu_id, 0 , 0, 0, 0);
}
void
machine_signal_idle_cancel(
processor_t processor)
{
cpu_signal_cancel(processor_to_cpu_datap(processor));
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_CANCEL_AST), processor->cpu_id, 0 , 0, 0, 0);
}
void
ml_install_interrupt_handler(
void *nub,
int source,
void *target,
IOInterruptHandler handler,
void *refCon)
{
cpu_data_t *cpu_data_ptr;
boolean_t current_state;
current_state = ml_set_interrupts_enabled(FALSE);
cpu_data_ptr = getCpuDatap();
cpu_data_ptr->interrupt_nub = nub;
cpu_data_ptr->interrupt_source = source;
cpu_data_ptr->interrupt_target = target;
cpu_data_ptr->interrupt_handler = handler;
cpu_data_ptr->interrupt_refCon = refCon;
cpu_data_ptr->interrupts_enabled = TRUE;
(void) ml_set_interrupts_enabled(current_state);
initialize_screen(NULL, kPEAcquireScreen);
}
void
ml_init_interrupt(void)
{
}
void ml_init_timebase(
void *args,
tbd_ops_t tbd_funcs,
vm_offset_t int_address,
vm_offset_t int_value __unused)
{
cpu_data_t *cpu_data_ptr;
cpu_data_ptr = (cpu_data_t *)args;
if ((cpu_data_ptr == &BootCpuData)
&& (rtclock_timebase_func.tbd_fiq_handler == (void *)NULL)) {
rtclock_timebase_func = *tbd_funcs;
rtclock_timebase_addr = int_address;
}
}
void
ml_parse_cpu_topology(void)
{
DTEntry entry, child __unused;
OpaqueDTEntryIterator iter;
uint32_t cpu_boot_arg;
int err;
cpu_boot_arg = MAX_CPUS;
PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg));
err = DTLookupEntry(NULL, "/cpus", &entry);
assert(err == kSuccess);
err = DTInitEntryIterator(entry, &iter);
assert(err == kSuccess);
while (kSuccess == DTIterateEntries(&iter, &child)) {
unsigned int propSize;
void *prop = NULL;
int cpu_id = avail_cpus++;
if (kSuccess == DTGetProperty(child, "cpu-id", &prop, &propSize))
cpu_id = *((int32_t*)prop);
assert(cpu_id < MAX_CPUS);
assert(cpu_phys_ids[cpu_id] == (uint32_t)-1);
if (boot_cpu == -1) {
if (kSuccess != DTGetProperty(child, "state", &prop, &propSize))
panic("unable to retrieve state for cpu %d", cpu_id);
if (strncmp((char*)prop, "running", propSize) == 0) {
boot_cpu = cpu_id;
}
}
if (kSuccess != DTGetProperty(child, "reg", &prop, &propSize))
panic("unable to retrieve physical ID for cpu %d", cpu_id);
cpu_phys_ids[cpu_id] = *((uint32_t*)prop);
if ((cpu_id > max_cpu_number) && ((cpu_id == boot_cpu) || (avail_cpus <= cpu_boot_arg)))
max_cpu_number = cpu_id;
}
if (avail_cpus > cpu_boot_arg)
avail_cpus = cpu_boot_arg;
if (avail_cpus == 0)
panic("No cpus found!");
if (boot_cpu == -1)
panic("unable to determine boot cpu!");
assert(__builtin_arm_rsr64("TPIDRRO_EL0") == 0);
__builtin_arm_wsr64("TPIDRRO_EL0", (uint64_t)boot_cpu);
}
unsigned int
ml_get_cpu_count(void)
{
return avail_cpus;
}
int
ml_get_boot_cpu_number(void)
{
return boot_cpu;
}
cluster_type_t
ml_get_boot_cluster(void)
{
return boot_cluster;
}
int
ml_get_cpu_number(uint32_t phys_id)
{
for (int log_id = 0; log_id <= ml_get_max_cpu_number(); ++log_id) {
if (cpu_phys_ids[log_id] == phys_id)
return log_id;
}
return -1;
}
int
ml_get_max_cpu_number(void)
{
return max_cpu_number;
}
void ml_lockdown_init() {
lockdown_handler_grp = lck_grp_alloc_init("lockdown_handler", NULL);
assert(lockdown_handler_grp != NULL);
lck_mtx_init(&lockdown_handler_lck, lockdown_handler_grp, NULL);
}
kern_return_t
ml_lockdown_handler_register(lockdown_handler_t f, void *this)
{
if (lockdown_handler || !f) {
return KERN_FAILURE;
}
lck_mtx_lock(&lockdown_handler_lck);
lockdown_handler = f;
lockdown_this = this;
#if !(defined(KERNEL_INTEGRITY_KTRR))
lockdown_done=1;
lockdown_handler(this);
#else
if (lockdown_done) {
lockdown_handler(this);
}
#endif
lck_mtx_unlock(&lockdown_handler_lck);
return KERN_SUCCESS;
}
void ml_lockdown_run_handler() {
lck_mtx_lock(&lockdown_handler_lck);
assert(!lockdown_done);
lockdown_done = 1;
if (lockdown_handler) {
lockdown_handler(lockdown_this);
}
lck_mtx_unlock(&lockdown_handler_lck);
}
kern_return_t
ml_processor_register(
ml_processor_info_t * in_processor_info,
processor_t * processor_out,
ipi_handler_t * ipi_handler)
{
cpu_data_t *this_cpu_datap;
processor_set_t pset;
boolean_t is_boot_cpu;
static unsigned int reg_cpu_count = 0;
if (in_processor_info->log_id > (uint32_t)ml_get_max_cpu_number())
return KERN_FAILURE;
if ((unsigned int)OSIncrementAtomic((SInt32*)®_cpu_count) >= avail_cpus)
return KERN_FAILURE;
if (in_processor_info->log_id != (uint32_t)ml_get_boot_cpu_number()) {
is_boot_cpu = FALSE;
this_cpu_datap = cpu_data_alloc(FALSE);
cpu_data_init(this_cpu_datap);
} else {
this_cpu_datap = &BootCpuData;
is_boot_cpu = TRUE;
}
assert(in_processor_info->log_id < MAX_CPUS);
this_cpu_datap->cpu_id = in_processor_info->cpu_id;
this_cpu_datap->cpu_console_buf = console_cpu_alloc(is_boot_cpu);
if (this_cpu_datap->cpu_console_buf == (void *)(NULL))
goto processor_register_error;
if (!is_boot_cpu) {
this_cpu_datap->cpu_number = in_processor_info->log_id;
if (cpu_data_register(this_cpu_datap) != KERN_SUCCESS)
goto processor_register_error;
}
this_cpu_datap->cpu_idle_notify = (void *) in_processor_info->processor_idle;
this_cpu_datap->cpu_cache_dispatch = in_processor_info->platform_cache_dispatch;
nanoseconds_to_absolutetime((uint64_t) in_processor_info->powergate_latency, &this_cpu_datap->cpu_idle_latency);
this_cpu_datap->cpu_reset_assist = kvtophys(in_processor_info->powergate_stub_addr);
this_cpu_datap->idle_timer_notify = (void *) in_processor_info->idle_timer;
this_cpu_datap->idle_timer_refcon = in_processor_info->idle_timer_refcon;
this_cpu_datap->platform_error_handler = (void *) in_processor_info->platform_error_handler;
this_cpu_datap->cpu_regmap_paddr = in_processor_info->regmap_paddr;
this_cpu_datap->cpu_phys_id = in_processor_info->phys_id;
this_cpu_datap->cpu_l2_access_penalty = in_processor_info->l2_access_penalty;
this_cpu_datap->cpu_cluster_type = in_processor_info->cluster_type;
this_cpu_datap->cpu_cluster_id = in_processor_info->cluster_id;
this_cpu_datap->cpu_l2_id = in_processor_info->l2_cache_id;
this_cpu_datap->cpu_l2_size = in_processor_info->l2_cache_size;
this_cpu_datap->cpu_l3_id = in_processor_info->l3_cache_id;
this_cpu_datap->cpu_l3_size = in_processor_info->l3_cache_size;
this_cpu_datap->cluster_master = is_boot_cpu;
pset = pset_find(in_processor_info->cluster_id, processor_pset(master_processor));
assert(pset != NULL);
kprintf("%s>cpu_id %p cluster_id %d cpu_number %d is type %d\n", __FUNCTION__, in_processor_info->cpu_id, in_processor_info->cluster_id, this_cpu_datap->cpu_number, in_processor_info->cluster_type);
if (!is_boot_cpu) {
processor_init((struct processor *)this_cpu_datap->cpu_processor,
this_cpu_datap->cpu_number, pset);
if (this_cpu_datap->cpu_l2_access_penalty) {
processor_set_primary(this_cpu_datap->cpu_processor,
master_processor);
}
}
*processor_out = this_cpu_datap->cpu_processor;
*ipi_handler = cpu_signal_handler;
if (in_processor_info->idle_tickle != (idle_tickle_t *) NULL)
*in_processor_info->idle_tickle = (idle_tickle_t) cpu_idle_tickle;
#if KPC
if (kpc_register_cpu(this_cpu_datap) != TRUE)
goto processor_register_error;
#endif
if (!is_boot_cpu) {
early_random_cpu_init(this_cpu_datap->cpu_number);
OSIncrementAtomic((SInt32*)&real_ncpus);
}
return KERN_SUCCESS;
processor_register_error:
#if KPC
kpc_unregister_cpu(this_cpu_datap);
#endif
if (!is_boot_cpu)
cpu_data_free(this_cpu_datap);
return KERN_FAILURE;
}
void
ml_init_arm_debug_interface(
void * in_cpu_datap,
vm_offset_t virt_address)
{
((cpu_data_t *)in_cpu_datap)->cpu_debug_interface_map = virt_address;
do_debugid();
}
void
init_ast_check(
__unused processor_t processor)
{
}
void
cause_ast_check(
processor_t processor)
{
if (current_processor() != processor) {
cpu_signal(processor_to_cpu_datap(processor), SIGPast, (void *)NULL, (void *)NULL);
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 1 , 0, 0, 0);
}
}
extern uint32_t cpu_idle_count;
void ml_get_power_state(boolean_t *icp, boolean_t *pidlep) {
*icp = ml_at_interrupt_context();
*pidlep = (cpu_idle_count == real_ncpus);
}
void
ml_cause_interrupt(void)
{
return;
}
vm_offset_t
ml_io_map(
vm_offset_t phys_addr,
vm_size_t size)
{
return (io_map(phys_addr, size, VM_WIMG_IO));
}
vm_offset_t
ml_io_map_wcomb(
vm_offset_t phys_addr,
vm_size_t size)
{
return (io_map(phys_addr, size, VM_WIMG_WCOMB));
}
vm_offset_t
ml_static_malloc(
__unused vm_size_t size)
{
return ((vm_offset_t) NULL);
}
vm_map_address_t
ml_map_high_window(
vm_offset_t phys_addr,
vm_size_t len)
{
return pmap_map_high_window_bd(phys_addr, len, VM_PROT_READ | VM_PROT_WRITE);
}
vm_offset_t
ml_static_ptovirt(
vm_offset_t paddr)
{
return phystokv(paddr);
}
vm_offset_t
ml_static_slide(
vm_offset_t vaddr)
{
return phystokv(vaddr + vm_kernel_slide - gVirtBase + gPhysBase);
}
vm_offset_t
ml_static_unslide(
vm_offset_t vaddr)
{
return (ml_static_vtop(vaddr) - gPhysBase + gVirtBase - vm_kernel_slide) ;
}
extern tt_entry_t *arm_kva_to_tte(vm_offset_t va);
kern_return_t
ml_static_protect(
vm_offset_t vaddr,
vm_size_t size,
vm_prot_t new_prot)
{
pt_entry_t arm_prot = 0;
pt_entry_t arm_block_prot = 0;
vm_offset_t vaddr_cur;
ppnum_t ppn;
kern_return_t result = KERN_SUCCESS;
if (vaddr < VM_MIN_KERNEL_ADDRESS) {
panic("ml_static_protect(): %p < %p", (void *) vaddr, (void *) VM_MIN_KERNEL_ADDRESS);
return KERN_FAILURE;
}
assert((vaddr & (PAGE_SIZE - 1)) == 0);
if ((new_prot & VM_PROT_WRITE) && (new_prot & VM_PROT_EXECUTE)) {
panic("ml_static_protect(): WX request on %p", (void *) vaddr);
}
if (new_prot & VM_PROT_WRITE) {
arm_prot |= ARM_PTE_AP(AP_RWNA);
arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RWNA);
} else {
arm_prot |= ARM_PTE_AP(AP_RONA);
arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RONA);
}
arm_prot |= ARM_PTE_NX;
arm_block_prot |= ARM_TTE_BLOCK_NX;
if (!(new_prot & VM_PROT_EXECUTE)) {
arm_prot |= ARM_PTE_PNX;
arm_block_prot |= ARM_TTE_BLOCK_PNX;
}
for (vaddr_cur = vaddr;
vaddr_cur < trunc_page_64(vaddr + size);
vaddr_cur += PAGE_SIZE) {
ppn = pmap_find_phys(kernel_pmap, vaddr_cur);
if (ppn != (vm_offset_t) NULL) {
tt_entry_t *tte2;
pt_entry_t *pte_p;
pt_entry_t ptmp;
tte2 = arm_kva_to_tte(vaddr_cur);
if (((*tte2) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) {
if ((((*tte2) & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) &&
((*tte2 & (ARM_TTE_BLOCK_NXMASK | ARM_TTE_BLOCK_PNXMASK | ARM_TTE_BLOCK_APMASK)) == arm_block_prot)) {
continue;
}
result = KERN_FAILURE;
break;
}
pte_p = (pt_entry_t *)&((tt_entry_t*)(phystokv((*tte2) & ARM_TTE_TABLE_MASK)))[(((vaddr_cur) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)];
ptmp = *pte_p;
if ((ptmp & ARM_PTE_HINT_MASK) && ((ptmp & (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) != arm_prot)) {
result = KERN_FAILURE;
break;
}
__unreachable_ok_push
if (TEST_PAGE_RATIO_4) {
{
unsigned int i;
pt_entry_t *ptep_iter;
ptep_iter = pte_p;
for (i=0; i<4; i++, ptep_iter++) {
ptmp = *ptep_iter;
if ((ptmp & (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) != arm_prot) {
ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) | arm_prot;
*ptep_iter = ptmp;
}
}
}
#ifndef __ARM_L1_PTW__
FlushPoC_DcacheRegion( trunc_page_32(pte_p), 4*sizeof(*pte_p));
#endif
} else {
ptmp = *pte_p;
if ((ptmp & (ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) != arm_prot) {
ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_PNXMASK | ARM_PTE_NXMASK)) | arm_prot;
*pte_p = ptmp;
}
#ifndef __ARM_L1_PTW__
FlushPoC_DcacheRegion( trunc_page_32(pte_p), sizeof(*pte_p));
#endif
}
__unreachable_ok_pop
}
}
if (vaddr_cur > vaddr) {
assert(((vaddr_cur - vaddr) & 0xFFFFFFFF00000000ULL) == 0);
flush_mmu_tlb_region(vaddr, (uint32_t)(vaddr_cur - vaddr));
}
return result;
}
void
ml_static_mfree(
vm_offset_t vaddr,
vm_size_t size)
{
vm_offset_t vaddr_cur;
ppnum_t ppn;
uint32_t freed_pages = 0;
if (vaddr < VM_MIN_KERNEL_ADDRESS)
return;
assert((vaddr & (PAGE_SIZE - 1)) == 0);
for (vaddr_cur = vaddr;
vaddr_cur < trunc_page_64(vaddr + size);
vaddr_cur += PAGE_SIZE) {
ppn = pmap_find_phys(kernel_pmap, vaddr_cur);
if (ppn != (vm_offset_t) NULL) {
if (ml_static_protect(vaddr_cur, PAGE_SIZE, VM_PROT_WRITE | VM_PROT_READ) != KERN_SUCCESS) {
panic("Failed ml_static_mfree on %p", (void *) vaddr_cur);
}
#if 0
#endif
vm_page_create(ppn, (ppn + 1));
freed_pages++;
}
}
vm_page_lockspin_queues();
vm_page_wire_count -= freed_pages;
vm_page_wire_count_initial -= freed_pages;
vm_page_unlock_queues();
#if DEBUG
kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn);
#endif
}
vm_offset_t
ml_vtophys(vm_offset_t vaddr)
{
return kvtophys(vaddr);
}
vm_size_t
ml_nofault_copy(vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size)
{
addr64_t cur_phys_dst, cur_phys_src;
vm_size_t count, nbytes = 0;
while (size > 0) {
if (!(cur_phys_src = kvtophys(virtsrc)))
break;
if (!(cur_phys_dst = kvtophys(virtdst)))
break;
if (!pmap_valid_address(trunc_page_64(cur_phys_dst)) ||
!pmap_valid_address(trunc_page_64(cur_phys_src)))
break;
count = PAGE_SIZE - (cur_phys_src & PAGE_MASK);
if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK)))
count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK);
if (count > size)
count = size;
bcopy_phys(cur_phys_src, cur_phys_dst, count);
nbytes += count;
virtsrc += count;
virtdst += count;
size -= count;
}
return nbytes;
}
boolean_t ml_validate_nofault(
vm_offset_t virtsrc, vm_size_t size)
{
addr64_t cur_phys_src;
uint32_t count;
while (size > 0) {
if (!(cur_phys_src = kvtophys(virtsrc)))
return FALSE;
if (!pmap_valid_address(trunc_page_64(cur_phys_src)))
return FALSE;
count = (uint32_t)(PAGE_SIZE - (cur_phys_src & PAGE_MASK));
if (count > size)
count = (uint32_t)size;
virtsrc += count;
size -= count;
}
return TRUE;
}
void
ml_get_bouncepool_info(vm_offset_t * phys_addr, vm_size_t * size)
{
*phys_addr = 0;
*size = 0;
}
void
active_rt_threads(__unused boolean_t active)
{
}
static void cpu_qos_cb_default(__unused int urgency, __unused uint64_t qos_param1, __unused uint64_t qos_param2) {
return;
}
cpu_qos_update_t cpu_qos_update = cpu_qos_cb_default;
void cpu_qos_update_register(cpu_qos_update_t cpu_qos_cb) {
if (cpu_qos_cb != NULL) {
cpu_qos_update = cpu_qos_cb;
} else {
cpu_qos_update = cpu_qos_cb_default;
}
}
void
thread_tell_urgency(int urgency, uint64_t rt_period, uint64_t rt_deadline, uint64_t sched_latency __unused, __unused thread_t nthread)
{
SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_START, urgency, rt_period, rt_deadline, sched_latency, 0);
cpu_qos_update(urgency, rt_period, rt_deadline);
SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_END, urgency, rt_period, rt_deadline, 0, 0);
}
void
machine_run_count(__unused uint32_t count)
{
}
processor_t
machine_choose_processor(__unused processor_set_t pset, processor_t processor)
{
return (processor);
}
#if KASAN
vm_offset_t ml_stack_base(void);
vm_size_t ml_stack_size(void);
vm_offset_t
ml_stack_base(void)
{
uintptr_t local = (uintptr_t) &local;
vm_offset_t intstack_top_ptr;
intstack_top_ptr = getCpuDatap()->intstack_top;
if ((local < intstack_top_ptr) && (local > intstack_top_ptr - INTSTACK_SIZE)) {
return intstack_top_ptr - INTSTACK_SIZE;
} else {
return current_thread()->kernel_stack;
}
}
vm_size_t
ml_stack_size(void)
{
uintptr_t local = (uintptr_t) &local;
vm_offset_t intstack_top_ptr;
intstack_top_ptr = getCpuDatap()->intstack_top;
if ((local < intstack_top_ptr) && (local > intstack_top_ptr - INTSTACK_SIZE)) {
return INTSTACK_SIZE;
} else {
return kernel_stack_size;
}
}
#endif
boolean_t machine_timeout_suspended(void) {
return FALSE;
}
kern_return_t
ml_interrupt_prewarm(__unused uint64_t deadline)
{
return KERN_FAILURE;
}
void
ml_set_decrementer(uint32_t dec_value)
{
cpu_data_t *cdp = getCpuDatap();
assert(ml_get_interrupts_enabled() == FALSE);
cdp->cpu_decrementer = dec_value;
if (cdp->cpu_set_decrementer_func) {
((void (*)(uint32_t))cdp->cpu_set_decrementer_func)(dec_value);
} else {
__asm__ volatile("msr CNTP_TVAL_EL0, %0" : : "r"((uint64_t)dec_value));
}
}
uint64_t ml_get_hwclock()
{
uint64_t timebase;
__asm__ volatile("isb\n"
"mrs %0, CNTPCT_EL0"
: "=r"(timebase));
return timebase;
}
uint64_t
ml_get_timebase()
{
return (ml_get_hwclock() + getCpuDatap()->cpu_base_timebase);
}
uint32_t
ml_get_decrementer()
{
cpu_data_t *cdp = getCpuDatap();
uint32_t dec;
assert(ml_get_interrupts_enabled() == FALSE);
if (cdp->cpu_get_decrementer_func) {
dec = ((uint32_t (*)(void))cdp->cpu_get_decrementer_func)();
} else {
uint64_t wide_val;
__asm__ volatile("mrs %0, CNTP_TVAL_EL0" : "=r"(wide_val));
dec = (uint32_t)wide_val;
assert(wide_val == (uint64_t)dec);
}
return dec;
}
boolean_t
ml_get_timer_pending()
{
uint64_t cntp_ctl;
__asm__ volatile("mrs %0, CNTP_CTL_EL0" : "=r"(cntp_ctl));
return ((cntp_ctl & CNTP_CTL_EL0_ISTATUS) != 0) ? TRUE : FALSE;
}
boolean_t
ml_wants_panic_trap_to_debugger(void)
{
boolean_t result = FALSE;
return result;
}
static void
cache_trap_error(thread_t thread, vm_map_address_t fault_addr)
{
mach_exception_data_type_t exc_data[2];
arm_saved_state_t *regs = get_user_regs(thread);
set_saved_state_far(regs, fault_addr);
exc_data[0] = KERN_INVALID_ADDRESS;
exc_data[1] = fault_addr;
exception_triage(EXC_BAD_ACCESS, exc_data, 2);
}
static void
cache_trap_recover()
{
vm_map_address_t fault_addr;
__asm__ volatile("mrs %0, FAR_EL1" : "=r"(fault_addr));
cache_trap_error(current_thread(), fault_addr);
}
static void
dcache_flush_trap(vm_map_address_t start, vm_map_size_t size)
{
vm_map_address_t end = start + size;
thread_t thread = current_thread();
vm_offset_t old_recover = thread->recover;
if (task_has_64Bit_addr(current_task())) {
if (end > MACH_VM_MAX_ADDRESS) {
cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1));
}
} else {
if (end > VM_MAX_ADDRESS) {
cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1));
}
}
if (start > end) {
cache_trap_error(thread, start & ((1 << ARM64_CLINE_SHIFT) - 1));
}
thread->recover = (vm_address_t)cache_trap_recover;
FlushPoC_DcacheRegion(start, (uint32_t)size);
thread->recover = old_recover;
}
static void
icache_invalidate_trap(vm_map_address_t start, vm_map_size_t size)
{
vm_map_address_t end = start + size;
thread_t thread = current_thread();
vm_offset_t old_recover = thread->recover;
if (task_has_64Bit_addr(current_task())) {
if (end > MACH_VM_MAX_ADDRESS) {
cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1));
}
} else {
if (end > VM_MAX_ADDRESS) {
cache_trap_error(thread, end & ((1 << ARM64_CLINE_SHIFT) - 1));
}
}
if (start > end) {
cache_trap_error(thread, start & ((1 << ARM64_CLINE_SHIFT) - 1));
}
thread->recover = (vm_address_t)cache_trap_recover;
CleanPoU_DcacheRegion(start, (uint32_t) size);
#if __ARM_IC_NOALIAS_ICACHE__
InvalidatePoU_IcacheRegion(start, (uint32_t)size);
#else
InvalidatePoU_Icache();
#endif
thread->recover = old_recover;
}
__attribute__((noreturn))
void
platform_syscall(arm_saved_state_t *state)
{
uint32_t code;
#define platform_syscall_kprintf(x...)
code = (uint32_t)get_saved_state_reg(state, 3);
switch (code) {
case 0:
platform_syscall_kprintf("icache flush requested.\n");
icache_invalidate_trap(get_saved_state_reg(state, 0), get_saved_state_reg(state, 1));
break;
case 1:
platform_syscall_kprintf("dcache flush requested.\n");
dcache_flush_trap(get_saved_state_reg(state, 0), get_saved_state_reg(state, 1));
break;
case 2:
platform_syscall_kprintf("set cthread self.\n");
thread_set_cthread_self(get_saved_state_reg(state, 0));
break;
case 3:
platform_syscall_kprintf("get cthread self.\n");
set_saved_state_reg(state, 0, thread_get_cthread_self());
break;
default:
platform_syscall_kprintf("unknown: %d\n", code);
break;
}
thread_exception_return();
}
static void
_enable_timebase_event_stream(uint32_t bit_index)
{
uint64_t cntkctl;
if (bit_index >= 64) {
panic("%s: invalid bit index (%u)", __FUNCTION__, bit_index);
}
__asm__ volatile ("mrs %0, CNTKCTL_EL1" : "=r"(cntkctl));
cntkctl |= (bit_index << CNTKCTL_EL1_EVENTI_SHIFT);
cntkctl |= CNTKCTL_EL1_EVNTEN;
cntkctl |= CNTKCTL_EL1_EVENTDIR;
if (user_timebase_allowed()) {
cntkctl |= CNTKCTL_EL1_PL0PCTEN;
}
__asm__ volatile ("msr CNTKCTL_EL1, %0" : : "r"(cntkctl));
}
static void
_enable_virtual_timer(void)
{
uint64_t cntvctl = CNTP_CTL_EL0_ENABLE;
__asm__ volatile ("msr CNTP_CTL_EL0, %0" : : "r"(cntvctl));
}
void
fiq_context_init(boolean_t enable_fiq __unused)
{
_enable_timebase_event_stream(fiq_eventi);
assert(ml_get_interrupts_enabled() == FALSE);
_enable_virtual_timer();
}
void
fiq_context_bootstrap(boolean_t enable_fiq)
{
#if defined(APPLE_ARM64_ARCH_FAMILY) || defined(BCM2837)
uint64_t ticks_per_sec, ticks_per_event, events_per_sec;
uint32_t bit_index;
ticks_per_sec = gPEClockFrequencyInfo.timebase_frequency_hz;
#if defined(ARM_BOARD_WFE_TIMEOUT_NS)
events_per_sec = 1000000000 / ARM_BOARD_WFE_TIMEOUT_NS;
#else
events_per_sec = 1000000;
#endif
ticks_per_event = ticks_per_sec / events_per_sec;
bit_index = flsll(ticks_per_event) - 1;
if ((ticks_per_event & ((1 << bit_index) - 1)) != 0)
bit_index++;
if (bit_index != 0)
bit_index--;
fiq_eventi = bit_index;
#else
#error Need a board configuration.
#endif
fiq_context_init(enable_fiq);
}
boolean_t
ml_delay_should_spin(uint64_t interval)
{
cpu_data_t *cdp = getCpuDatap();
if (cdp->cpu_idle_latency) {
return (interval < cdp->cpu_idle_latency) ? TRUE : FALSE;
} else {
return FALSE;
}
}
void
ml_delay_on_yield(void)
{
}
boolean_t ml_thread_is64bit(thread_t thread) {
return (thread_is_64bit_addr(thread));
}
void ml_timer_evaluate(void) {
}
boolean_t
ml_timer_forced_evaluation(void) {
return FALSE;
}
uint64_t
ml_energy_stat(thread_t t) {
return t->machine.energy_estimate_nj;
}
void
ml_gpu_stat_update(__unused uint64_t gpu_ns_delta) {
#if CONFIG_EMBEDDED
task_coalition_update_gpu_stats(current_task(), gpu_ns_delta);
#endif
}
uint64_t
ml_gpu_stat(__unused thread_t t) {
return 0;
}
#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
static void
timer_state_event(boolean_t switch_to_kernel)
{
thread_t thread = current_thread();
if (!thread->precise_user_kernel_time) return;
processor_data_t *pd = &getCpuDatap()->cpu_processor->processor_data;
uint64_t now = ml_get_timebase();
timer_stop(pd->current_state, now);
pd->current_state = (switch_to_kernel) ? &pd->system_state : &pd->user_state;
timer_start(pd->current_state, now);
timer_stop(pd->thread_timer, now);
pd->thread_timer = (switch_to_kernel) ? &thread->system_timer : &thread->user_timer;
timer_start(pd->thread_timer, now);
}
void
timer_state_event_user_to_kernel(void)
{
timer_state_event(TRUE);
}
void
timer_state_event_kernel_to_user(void)
{
timer_state_event(FALSE);
}
#endif
extern thread_t current_act(void);
thread_t
current_act(void)
{
return current_thread_fast();
}
#undef current_thread
extern thread_t current_thread(void);
thread_t
current_thread(void)
{
return current_thread_fast();
}
typedef struct
{
ex_cb_t cb;
void *refcon;
}
ex_cb_info_t;
ex_cb_info_t ex_cb_info[EXCB_CLASS_MAX];
kern_return_t ex_cb_register(
ex_cb_class_t cb_class,
ex_cb_t cb,
void *refcon)
{
ex_cb_info_t *pInfo = &ex_cb_info[cb_class];
if ((NULL == cb) || (cb_class >= EXCB_CLASS_MAX))
{
return KERN_INVALID_VALUE;
}
if (NULL == pInfo->cb)
{
pInfo->cb = cb;
pInfo->refcon = refcon;
return KERN_SUCCESS;
}
return KERN_FAILURE;
}
ex_cb_action_t ex_cb_invoke(
ex_cb_class_t cb_class,
vm_offset_t far)
{
ex_cb_info_t *pInfo = &ex_cb_info[cb_class];
ex_cb_state_t state = {far};
if (cb_class >= EXCB_CLASS_MAX)
{
panic("Invalid exception callback class 0x%x\n", cb_class);
}
if (pInfo->cb)
{
return pInfo->cb(cb_class, pInfo->refcon, &state);
}
return EXCB_ACTION_NONE;
}