#include <mach/i386/vm_param.h>
#include <string.h>
#include <stdint.h>
#include <mach/vm_param.h>
#include <mach/vm_prot.h>
#include <mach/machine.h>
#include <mach/time_value.h>
#include <kern/spl.h>
#include <kern/assert.h>
#include <kern/debug.h>
#include <kern/misc_protos.h>
#include <kern/startup.h>
#include <kern/clock.h>
#include <kern/pms.h>
#include <kern/cpu_data.h>
#include <kern/processor.h>
#include <sys/kdebug.h>
#include <console/serial_protos.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <machine/pal_routines.h>
#include <i386/fpu.h>
#include <i386/pmap.h>
#include <i386/misc_protos.h>
#include <i386/cpu_threads.h>
#include <i386/cpuid.h>
#include <i386/lapic.h>
#include <i386/mp.h>
#include <i386/mp_desc.h>
#if CONFIG_MTRR
#include <i386/mtrr.h>
#endif
#include <i386/machine_routines.h>
#if CONFIG_MCA
#include <i386/machine_check.h>
#endif
#include <i386/ucode.h>
#include <i386/postcode.h>
#include <i386/Diagnostics.h>
#include <i386/pmCPU.h>
#include <i386/tsc.h>
#include <i386/locks.h>
#if DEBUG
#include <machine/pal_routines.h>
#endif
#if MONOTONIC
#include <kern/monotonic.h>
#endif
#include <san/kasan.h>
#if DEBUG
#define DBG(x ...) kprintf(x)
#else
#define DBG(x ...)
#endif
int debug_task;
int early_boot = 1;
static boot_args *kernelBootArgs;
extern int disableConsoleOutput;
extern const char version[];
extern const char version_variant[];
extern int nx_enabled;
uint64_t physmap_base = 0;
uint64_t physmap_max = 4 * GB;
pd_entry_t *KPTphys;
pd_entry_t *IdlePTD;
pdpt_entry_t *IdlePDPT;
pml4_entry_t *IdlePML4;
int kernPhysPML4Index;
int kernPhysPML4EntryCount;
ppnum_t released_PT_ppn = 0;
uint32_t released_PT_cnt = 0;
char *physfree;
void idt64_remap(void);
static void *
ALLOCPAGES(int npages)
{
uintptr_t tmp = (uintptr_t)physfree;
bzero(physfree, npages * PAGE_SIZE);
physfree += npages * PAGE_SIZE;
tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;
return (void *)tmp;
}
static void
fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
{
int i;
for (i = 0; i < count; i++) {
base[index] = src | prot | INTEL_PTE_VALID;
src += PAGE_SIZE;
index++;
}
}
extern pmap_paddr_t first_avail;
int break_kprintf = 0;
uint64_t
x86_64_pre_sleep(void)
{
IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];
uint64_t oldcr3 = get_cr3_raw();
set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));
return oldcr3;
}
void
x86_64_post_sleep(uint64_t new_cr3)
{
IdlePML4[0] = 0;
set_cr3_raw((uint32_t) new_cr3);
}
static int
physmap_init_L2(uint64_t *physStart, pt_entry_t **l2ptep)
{
unsigned i;
pt_entry_t *physmapL2 = ALLOCPAGES(1);
if (physmapL2 == NULL) {
DBG("physmap_init_L2 page alloc failed when initting L2 for physAddr 0x%llx.\n", *physStart);
*l2ptep = NULL;
return -1;
}
for (i = 0; i < NPDPG; i++) {
physmapL2[i] = *physStart
| INTEL_PTE_PS
| INTEL_PTE_VALID
| INTEL_PTE_NX
| INTEL_PTE_WRITE;
*physStart += NBPD;
}
*l2ptep = physmapL2;
return 0;
}
static int
physmap_init_L3(int startIndex, uint64_t highest_phys, uint64_t *physStart, pt_entry_t **l3ptep)
{
unsigned i;
int ret;
pt_entry_t *l2pte;
pt_entry_t *physmapL3 = ALLOCPAGES(1);
if (physmapL3 == NULL) {
DBG("physmap_init_L3 page alloc failed when initting L3 for physAddr 0x%llx.\n", *physStart);
*l3ptep = NULL;
return -1;
}
for (i = startIndex; i < NPDPTPG && *physStart < highest_phys; i++) {
if ((ret = physmap_init_L2(physStart, &l2pte)) < 0) {
return ret;
}
physmapL3[i] = ((uintptr_t)ID_MAP_VTOP(l2pte))
| INTEL_PTE_VALID
| INTEL_PTE_NX
| INTEL_PTE_WRITE;
}
*l3ptep = physmapL3;
return 0;
}
static void
physmap_init(uint8_t phys_random_L3, uint64_t *new_physmap_base, uint64_t *new_physmap_max)
{
pt_entry_t *l3pte;
int pml4_index, i;
int L3_start_index;
uint64_t physAddr = 0;
uint64_t highest_physaddr;
unsigned pdpte_count;
#if DEVELOPMENT || DEBUG
if (kernelBootArgs->PhysicalMemorySize > K64_MAXMEM) {
panic("Installed physical memory exceeds configured maximum.");
}
#endif
highest_physaddr = kernelBootArgs->PhysicalMemorySize + 4 * GB;
assert(highest_physaddr < (UINT64_MAX - PDPTMASK));
pdpte_count = (unsigned) (((highest_physaddr + PDPTMASK) >> PDPTSHIFT) + phys_random_L3);
kernPhysPML4EntryCount = (pdpte_count + ((1U << (PML4SHIFT - PDPTSHIFT)) - 1)) >> (PML4SHIFT - PDPTSHIFT);
if (kernPhysPML4EntryCount == 0) {
kernPhysPML4EntryCount = 1;
}
if (kernPhysPML4EntryCount > KERNEL_PHYSMAP_PML4_COUNT_MAX) {
#if DEVELOPMENT || DEBUG
panic("physmap too large");
#else
kprintf("[pmap] Limiting physmap to %d PML4s (was %d)\n", KERNEL_PHYSMAP_PML4_COUNT_MAX,
kernPhysPML4EntryCount);
kernPhysPML4EntryCount = KERNEL_PHYSMAP_PML4_COUNT_MAX;
#endif
}
kernPhysPML4Index = KERNEL_KEXTS_INDEX - kernPhysPML4EntryCount;
#if DEVELOPMENT || DEBUG
uint32_t reg[4];
do_cpuid(0x80000000, reg);
if (reg[eax] >= 0x80000001) {
do_cpuid(0x80000001, reg);
assert(reg[edx] & CPUID_EXTFEATURE_XD);
}
#endif
L3_start_index = phys_random_L3;
for (pml4_index = kernPhysPML4Index;
pml4_index < (kernPhysPML4Index + kernPhysPML4EntryCount) && physAddr < highest_physaddr;
pml4_index++) {
if (physmap_init_L3(L3_start_index, highest_physaddr, &physAddr, &l3pte) < 0) {
panic("Physmap page table initialization failed");
}
L3_start_index = 0;
IdlePML4[pml4_index] = ((uintptr_t)ID_MAP_VTOP(l3pte))
| INTEL_PTE_VALID
| INTEL_PTE_NX
| INTEL_PTE_WRITE;
}
*new_physmap_base = KVADDR(kernPhysPML4Index, phys_random_L3, 0, 0);
*new_physmap_max = *new_physmap_base + physAddr;
DBG("Physical address map base: 0x%qx\n", *new_physmap_base);
for (i = kernPhysPML4Index; i < (kernPhysPML4Index + kernPhysPML4EntryCount); i++) {
DBG("Physical map idlepml4[%d]: 0x%llx\n", i, IdlePML4[i]);
}
}
void doublemap_init(uint8_t);
static void
Idle_PTs_init(void)
{
uint64_t rand64;
uint64_t new_physmap_base, new_physmap_max;
KPTphys = ALLOCPAGES(NKPT);
IdlePTD = ALLOCPAGES(NPGPTD);
IdlePDPT = ALLOCPAGES(1);
IdlePML4 = ALLOCPAGES(1);
fillkpt(KPTphys,
INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
fillkpt(IdlePTD,
INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
fillkpt(IdlePDPT,
INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
fillkpt(IdlePML4 + KERNEL_PML4_INDEX,
INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
postcode(VSTART_PHYSMAP_INIT);
rand64 = early_random();
physmap_init(rand64 & 0xFF, &new_physmap_base, &new_physmap_max);
doublemap_init((rand64 >> 8) & 0xFF);
idt64_remap();
postcode(VSTART_SET_CR3);
physmap_base = new_physmap_base;
physmap_max = new_physmap_max;
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
}
void
Idle_PTs_release(vm_offset_t start, vm_offset_t end)
{
uint32_t i;
uint32_t index_start;
uint32_t index_limit;
ppnum_t pn_first;
ppnum_t pn;
uint32_t cnt;
start = ((start + I386_LPGMASK) & ~I386_LPGMASK);
index_start = (uint32_t)((start - KERNEL_BASE) >> PAGE_SHIFT);
index_limit = (uint32_t)((end - KERNEL_BASE) >> PAGE_SHIFT);
if (index_limit > NKPT * PTE_PER_PAGE) {
index_limit = NKPT * PTE_PER_PAGE;
}
for (i = index_start; i < index_limit; ++i) {
assert(KPTphys[i] == 0);
if (KPTphys[i] != 0) {
return;
}
}
index_start >>= PTPGSHIFT;
index_limit >>= PTPGSHIFT;
if (index_limit > NPGPTD * PTE_PER_PAGE) {
index_limit = NPGPTD * PTE_PER_PAGE;
}
if (index_limit <= index_start) {
return;
}
pn_first = 0;
cnt = 0;
for (i = index_start; i < index_limit; ++i) {
assert(IdlePTD[i] != 0);
if (IdlePTD[i] == 0) {
return;
}
pn = (ppnum_t)((PG_FRAME & IdlePTD[i]) >> PTSHIFT);
if (cnt == 0) {
pn_first = pn;
} else {
assert(pn == pn_first + cnt);
if (pn != pn_first + cnt) {
return;
}
}
++cnt;
}
for (i = index_start; i < index_limit; ++i) {
IdlePTD[i] = 0;
}
set_cr3_raw(get_cr3_raw());
released_PT_ppn = pn_first;
released_PT_cnt = cnt;
#if DEVELOPMENT || DEBUG
printf("Idle_PTs_release %d pages from PFN 0x%x\n", released_PT_cnt, released_PT_ppn);
#endif
}
extern void vstart_trap_handler;
#define BOOT_TRAP_VECTOR(t) \
[t] = { \
(uintptr_t) &vstart_trap_handler, \
KERNEL64_CS, \
0, \
ACC_P|ACC_PL_K|ACC_INTR_GATE, \
0 \
},
#define L0(x, n) x(n)
#define L1(x, n) L0(x,n-1) L0(x,n)
#define L2(x, n) L1(x,n-2) L1(x,n)
#define L3(x, n) L2(x,n-4) L2(x,n)
#define L4(x, n) L3(x,n-8) L3(x,n)
#define L5(x, n) L4(x,n-16) L4(x,n)
#define FOR_0_TO_31(x) L5(x,31)
struct fake_descriptor64 master_boot_idt64[IDTSZ]
__attribute__((section("__HIB,__desc")))
__attribute__((aligned(PAGE_SIZE))) = {
FOR_0_TO_31(BOOT_TRAP_VECTOR)
};
static void
vstart_idt_init(void)
{
x86_64_desc_register_t vstart_idt = {
sizeof(master_boot_idt64),
master_boot_idt64
};
fix_desc64(master_boot_idt64, 32);
lidt((void *)&vstart_idt);
}
__attribute__((noreturn))
void
vstart(vm_offset_t boot_args_start)
{
boolean_t is_boot_cpu = !(boot_args_start == 0);
int cpu = 0;
uint32_t lphysfree;
#if DEBUG
uint64_t gsbase;
#endif
postcode(VSTART_ENTRY);
if (is_boot_cpu) {
vstart_idt_init();
postcode(VSTART_IDT_INIT);
wrmsr64(MSR_IA32_GS_BASE, EARLY_GSBASE_MAGIC);
kernelBootArgs = (boot_args *)boot_args_start;
lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1));
pal_serial_init();
DBG("revision 0x%x\n", kernelBootArgs->Revision);
DBG("version 0x%x\n", kernelBootArgs->Version);
DBG("command line %s\n", kernelBootArgs->CommandLine);
DBG("memory map 0x%x\n", kernelBootArgs->MemoryMap);
DBG("memory map sz 0x%x\n", kernelBootArgs->MemoryMapSize);
DBG("kaddr 0x%x\n", kernelBootArgs->kaddr);
DBG("ksize 0x%x\n", kernelBootArgs->ksize);
DBG("physfree %p\n", physfree);
DBG("bootargs: %p, &ksize: %p &kaddr: %p\n",
kernelBootArgs,
&kernelBootArgs->ksize,
&kernelBootArgs->kaddr);
DBG("SMBIOS mem sz 0x%llx\n", kernelBootArgs->PhysicalMemorySize);
kernelBootArgs = (boot_args *)
ml_static_ptovirt(boot_args_start);
DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
(unsigned long)boot_args_start, kernelBootArgs);
#if KASAN
kasan_reserve_memory(kernelBootArgs);
#endif
PE_init_platform(FALSE, kernelBootArgs);
postcode(PE_INIT_PLATFORM_D);
Idle_PTs_init();
postcode(VSTART_IDLE_PTS_INIT);
#if KASAN
kasan_init();
kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree));
#endif
#if MONOTONIC
mt_early_init();
#endif
first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
cpu_data_alloc(TRUE);
cpu_desc_init(cpu_datap(0));
postcode(VSTART_CPU_DESC_INIT);
cpu_desc_load(cpu_datap(0));
postcode(VSTART_CPU_MODE_INIT);
cpu_syscall_init(cpu_datap(0));
} else {
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
cpu = lapic_to_cpu[(LAPIC_READ(ID) >> LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
#if DEBUG
gsbase = rdmsr64(MSR_IA32_GS_BASE);
#endif
cpu_desc_load(cpu_datap(cpu));
DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, gsbase);
}
early_boot = 0;
postcode(VSTART_EXIT);
x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
: (uintptr_t) i386_init_slave,
cpu_datap(cpu)->cpu_int_stack_top);
}
void
pstate_trace(void)
{
}
void
i386_init(void)
{
unsigned int maxmem;
uint64_t maxmemtouse;
unsigned int cpus = 0;
boolean_t fidn;
boolean_t IA32e = TRUE;
postcode(I386_INIT_ENTRY);
pal_i386_init();
tsc_init();
rtclock_early_init();
kernel_debug_string_early("i386_init");
pstate_trace();
#if CONFIG_MCA
mca_cpu_init();
#endif
master_cpu = 0;
lck_mod_init();
printf_init();
timer_call_init();
cpu_init();
postcode(CPU_INIT_D);
panic_init();
kernel_debug_string_early("PE_init_kprintf");
PE_init_kprintf(FALSE);
kernel_debug_string_early("kernel_early_bootstrap");
kernel_early_bootstrap();
if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof(dgWork.dgFlags))) {
dgWork.dgFlags = 0;
}
serialmode = 0;
if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
kprintf("Serial mode specified: %08X\n", serialmode);
int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
if (force_sync) {
serialmode |= SERIALMODE_SYNCDRAIN;
kprintf(
"WARNING: Forcing uart driver to output synchronously."
"printf()s/IOLogs will impact kernel performance.\n"
"You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
}
}
}
if (serialmode & SERIALMODE_OUTPUT) {
(void)switch_to_serial_console();
disableConsoleOutput = FALSE;
}
kernel_debug_string_early("PE_init_printf");
PE_init_printf(FALSE);
kprintf("version_variant = %s\n", version_variant);
kprintf("version = %s\n", version);
if (!PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
maxmemtouse = 0;
} else {
maxmemtouse = ((uint64_t)maxmem) * MB;
}
if (PE_parse_boot_argn("cpus", &cpus, sizeof(cpus))) {
if ((0 < cpus) && (cpus < max_ncpus)) {
max_ncpus = cpus;
}
}
PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof(vm_himemory_mode));
if (!vm_himemory_mode) {
kprintf("himemory_mode disabled\n");
}
if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof(fidn))) {
force_immediate_debugger_NMI = FALSE;
} else {
force_immediate_debugger_NMI = fidn;
}
#if DEBUG
nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold);
#endif
PE_parse_boot_argn("urgency_notification_abstime",
&urgency_notification_assert_abstime_threshold,
sizeof(urgency_notification_assert_abstime_threshold));
if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD)) {
nx_enabled = 0;
}
kernel_debug_string_early("i386_vm_init");
i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
PE_init_platform(TRUE, kernelBootArgs);
PE_create_console();
kernel_debug_string_early("power_management_init");
power_management_init();
#if MONOTONIC
mt_cpu_up(cpu_datap(0));
#endif
processor_bootstrap();
thread_bootstrap();
pstate_trace();
kernel_debug_string_early("machine_startup");
machine_startup();
pstate_trace();
}
static void __dead2
do_init_slave(boolean_t fast_restart)
{
void *init_param = FULL_SLAVE_INIT;
postcode(I386_INIT_SLAVE);
if (!fast_restart) {
set_cr0(get_cr0() & ~(CR0_NW | CR0_CD));
DBG("i386_init_slave() CPU%d: phys (%d) active.\n",
get_cpu_number(), get_cpu_phys_number());
assert(!ml_get_interrupts_enabled());
cpu_syscall_init(current_cpu_datap());
pmap_cpu_init();
#if CONFIG_MCA
mca_cpu_init();
#endif
LAPIC_INIT();
lapic_configure();
LAPIC_DUMP();
LAPIC_CPU_MAP_DUMP();
init_fpu();
#if CONFIG_MTRR
mtrr_update_cpu();
#endif
ucode_update_wake();
} else {
init_param = FAST_SLAVE_INIT;
}
#if CONFIG_VMX
vmx_resume(FALSE);
#endif
#if CONFIG_MTRR
if (!fast_restart) {
pat_init();
}
#endif
cpu_thread_init();
cpu_init();
#if MONOTONIC
mt_cpu_up(current_cpu_datap());
#endif
slave_main(init_param);
panic("do_init_slave() returned from slave_main()");
}
void
i386_init_slave(void)
{
do_init_slave(FALSE);
}
void
i386_init_slave_fast(void)
{
do_init_slave(TRUE);
}
#include <libkern/kernel_mach_header.h>
uint64_t dblmap_base, dblmap_max;
kernel_segment_command_t *hdescseg;
pt_entry_t *dblmapL3;
unsigned int dblallocs;
uint64_t dblmap_dist;
extern uint64_t idt64_hndl_table0[];
void
doublemap_init(uint8_t randL3)
{
dblmapL3 = ALLOCPAGES(1); dblallocs++;
struct {
pt_entry_t entries[PTE_PER_PAGE];
} * dblmapL2 = ALLOCPAGES(1); dblallocs++;
dblmapL3[randL3] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
| INTEL_PTE_VALID
| INTEL_PTE_WRITE;
hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB");
vm_offset_t hdescb = hdescseg->vmaddr;
unsigned long hdescsz = hdescseg->vmsize;
unsigned long hdescszr = round_page_64(hdescsz);
vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr;
kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text");
vm_offset_t thdescb = thdescsect->addr;
unsigned long thdescsz = thdescsect->size;
unsigned long thdescszr = round_page_64(thdescsz);
vm_offset_t thdesce = thdescb + thdescszr;
assert((hdescb & 0xFFF) == 0);
for (int i = 0; hdescc < hdesce; i++) {
struct {
pt_entry_t entries[PTE_PER_PAGE];
} * dblmapL1 = ALLOCPAGES(1);
dblallocs++;
dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE);
for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) {
uint64_t template = INTEL_PTE_VALID;
if ((hdescc >= thdescb) && (hdescc < thdesce)) {
} else {
template |= INTEL_PTE_WRITE | INTEL_PTE_NX;
}
dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template;
hdescc += PAGE_SIZE;
}
}
IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, randL3, 0, 0);
dblmap_max = dblmap_base + hdescszr;
dblmap_dist = dblmap_base - hdescb;
idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]);
idt64_hndl_table0[3] = DBLMAP(idt64_hndl_table0[3]);
idt64_hndl_table0[6] = (uint64_t)(uintptr_t)&kernel_stack_mask;
extern cpu_data_t cpshadows[], scdatas[];
uintptr_t cd1 = (uintptr_t) &cpshadows[0];
uintptr_t cd2 = (uintptr_t) &scdatas[0];
idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2;
DBG("Double map base: 0x%qx\n", dblmap_base);
DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]);
assert(LDTSZ > LDTSZ_MIN);
}
vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
#include <i386/pmap_internal.h>
vm_offset_t
dyn_dblmap(vm_offset_t cva, vm_offset_t sz)
{
vm_offset_t ava = dblmap_max;
assert((sz & PAGE_MASK) == 0);
assert(cva != 0);
pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP);
dblmap_max += sz;
return ava - cva;
}
void
idt64_remap(void)
{
for (int i = 0; i < IDTSZ; i++) {
master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64);
}
}