#include <mach/i386/vm_param.h>
#include <string.h>
#include <mach/vm_param.h>
#include <mach/vm_prot.h>
#include <mach/machine.h>
#include <mach/time_value.h>
#include <kern/spl.h>
#include <kern/assert.h>
#include <kern/debug.h>
#include <kern/misc_protos.h>
#include <kern/startup.h>
#include <kern/clock.h>
#include <kern/pms.h>
#include <kern/xpr.h>
#include <kern/cpu_data.h>
#include <kern/processor.h>
#include <sys/kdebug.h>
#include <console/serial_protos.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <machine/pal_routines.h>
#include <i386/fpu.h>
#include <i386/pmap.h>
#include <i386/misc_protos.h>
#include <i386/cpu_threads.h>
#include <i386/cpuid.h>
#include <i386/lapic.h>
#include <i386/mp.h>
#include <i386/mp_desc.h>
#if CONFIG_MTRR
#include <i386/mtrr.h>
#endif
#include <i386/machine_routines.h>
#if CONFIG_MCA
#include <i386/machine_check.h>
#endif
#include <i386/ucode.h>
#include <i386/postcode.h>
#include <i386/Diagnostics.h>
#include <i386/pmCPU.h>
#include <i386/tsc.h>
#include <i386/locks.h>
#if DEBUG
#include <machine/pal_routines.h>
#endif
#if DEBUG
#define DBG(x...) kprintf(x)
#else
#define DBG(x...)
#endif
int debug_task;
static boot_args *kernelBootArgs;
extern int disableConsoleOutput;
extern const char version[];
extern const char version_variant[];
extern int nx_enabled;
uint64_t physmap_base, physmap_max;
pd_entry_t *KPTphys;
pd_entry_t *IdlePTD;
pdpt_entry_t *IdlePDPT;
pml4_entry_t *IdlePML4;
char *physfree;
static void *
ALLOCPAGES(int npages)
{
uintptr_t tmp = (uintptr_t)physfree;
bzero(physfree, npages * PAGE_SIZE);
physfree += npages * PAGE_SIZE;
tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;
return (void *)tmp;
}
static void
fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
{
int i;
for (i=0; i<count; i++) {
base[index] = src | prot | INTEL_PTE_VALID;
src += PAGE_SIZE;
index++;
}
}
extern pmap_paddr_t first_avail;
int break_kprintf = 0;
uint64_t
x86_64_pre_sleep(void)
{
IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];
uint64_t oldcr3 = get_cr3_raw();
set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));
return oldcr3;
}
void
x86_64_post_sleep(uint64_t new_cr3)
{
IdlePML4[0] = 0;
set_cr3_raw((uint32_t) new_cr3);
}
extern int maxphymapsupported[NPHYSMAP <= (PTE_PER_PAGE/2) ? 1 : -1];
static void
physmap_init(void)
{
pt_entry_t *physmapL3 = ALLOCPAGES(1);
struct {
pt_entry_t entries[PTE_PER_PAGE];
} * physmapL2 = ALLOCPAGES(NPHYSMAP);
uint64_t i;
uint8_t phys_random_L3 = early_random() & 0xFF;
#if DEVELOPMENT || DEBUG
uint32_t reg[4];
do_cpuid(0x80000000, reg);
if (reg[eax] >= 0x80000001) {
do_cpuid(0x80000001, reg);
assert(reg[edx] & CPUID_EXTFEATURE_XD);
}
#endif
for(i = 0; i < NPHYSMAP; i++) {
physmapL3[i + phys_random_L3] =
((uintptr_t)ID_MAP_VTOP(&physmapL2[i]))
| INTEL_PTE_VALID
| INTEL_PTE_NX
| INTEL_PTE_WRITE;
uint64_t j;
for(j = 0; j < PTE_PER_PAGE; j++) {
physmapL2[i].entries[j] =
((i * PTE_PER_PAGE + j) << PDSHIFT)
| INTEL_PTE_PS
| INTEL_PTE_VALID
| INTEL_PTE_NX
| INTEL_PTE_WRITE;
}
}
IdlePML4[KERNEL_PHYSMAP_PML4_INDEX] =
((uintptr_t)ID_MAP_VTOP(physmapL3))
| INTEL_PTE_VALID
| INTEL_PTE_NX
| INTEL_PTE_WRITE;
physmap_base = KVADDR(KERNEL_PHYSMAP_PML4_INDEX, phys_random_L3, 0, 0);
physmap_max = physmap_base + NPHYSMAP * GB;
DBG("Physical address map base: 0x%qx\n", physmap_base);
DBG("Physical map idlepml4[%d]: 0x%llx\n",
KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]);
}
static void
descriptor_alias_init()
{
vm_offset_t master_gdt_phys;
vm_offset_t master_gdt_alias_phys;
vm_offset_t master_idt_phys;
vm_offset_t master_idt_alias_phys;
assert(((vm_offset_t)master_gdt & PAGE_MASK) == 0);
assert(((vm_offset_t)master_idt64 & PAGE_MASK) == 0);
master_gdt_phys = (vm_offset_t) ID_MAP_VTOP(master_gdt);
master_idt_phys = (vm_offset_t) ID_MAP_VTOP(master_idt64);
master_gdt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_GDT_ALIAS);
master_idt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_IDT_ALIAS);
DBG("master_gdt_phys: %p\n", (void *) master_gdt_phys);
DBG("master_idt_phys: %p\n", (void *) master_idt_phys);
DBG("master_gdt_alias_phys: %p\n", (void *) master_gdt_alias_phys);
DBG("master_idt_alias_phys: %p\n", (void *) master_idt_alias_phys);
KPTphys[atop_kernel(master_gdt_alias_phys)] = master_gdt_phys |
INTEL_PTE_VALID | INTEL_PTE_NX | INTEL_PTE_WRITE;
KPTphys[atop_kernel(master_idt_alias_phys)] = master_idt_phys |
INTEL_PTE_VALID | INTEL_PTE_NX;
}
static void
Idle_PTs_init(void)
{
KPTphys = ALLOCPAGES(NKPT);
IdlePTD = ALLOCPAGES(NPGPTD);
IdlePDPT = ALLOCPAGES(1);
IdlePML4 = ALLOCPAGES(1);
fillkpt(KPTphys,
INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
fillkpt(IdlePTD,
INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
fillkpt(IdlePDPT,
INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
fillkpt(IdlePML4 + KERNEL_PML4_INDEX,
INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
postcode(VSTART_PHYSMAP_INIT);
physmap_init();
postcode(VSTART_DESC_ALIAS_INIT);
descriptor_alias_init();
postcode(VSTART_SET_CR3);
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
}
void
vstart(vm_offset_t boot_args_start)
{
boolean_t is_boot_cpu = !(boot_args_start == 0);
int cpu;
uint32_t lphysfree;
postcode(VSTART_ENTRY);
if (is_boot_cpu) {
kernelBootArgs = (boot_args *)boot_args_start;
lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) &~ (PAGE_SIZE - 1));
#if DEBUG
pal_serial_init();
#endif
DBG("revision 0x%x\n", kernelBootArgs->Revision);
DBG("version 0x%x\n", kernelBootArgs->Version);
DBG("command line %s\n", kernelBootArgs->CommandLine);
DBG("memory map 0x%x\n", kernelBootArgs->MemoryMap);
DBG("memory map sz 0x%x\n", kernelBootArgs->MemoryMapSize);
DBG("kaddr 0x%x\n", kernelBootArgs->kaddr);
DBG("ksize 0x%x\n", kernelBootArgs->ksize);
DBG("physfree %p\n", physfree);
DBG("bootargs: %p, &ksize: %p &kaddr: %p\n",
kernelBootArgs,
&kernelBootArgs->ksize,
&kernelBootArgs->kaddr);
kernelBootArgs = (boot_args *)
ml_static_ptovirt(boot_args_start);
DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
(unsigned long)boot_args_start, kernelBootArgs);
PE_init_platform(FALSE, kernelBootArgs);
postcode(PE_INIT_PLATFORM_D);
Idle_PTs_init();
postcode(VSTART_IDLE_PTS_INIT);
first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
cpu = 0;
cpu_data_alloc(TRUE);
} else {
set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
cpu = lapic_to_cpu[(LAPIC_READ(ID)>>LAPIC_ID_SHIFT) & LAPIC_ID_MASK];
DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, rdmsr64(MSR_IA32_GS_BASE));
}
postcode(VSTART_CPU_DESC_INIT);
if(is_boot_cpu)
cpu_desc_init64(cpu_datap(cpu));
cpu_desc_load64(cpu_datap(cpu));
postcode(VSTART_CPU_MODE_INIT);
if (is_boot_cpu)
cpu_mode_init(current_cpu_datap());
postcode(VSTART_EXIT);
x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
: (uintptr_t) i386_init_slave,
cpu_datap(cpu)->cpu_int_stack_top);
}
void
pstate_trace(void)
{
}
void
i386_init(void)
{
unsigned int maxmem;
uint64_t maxmemtouse;
unsigned int cpus = 0;
boolean_t fidn;
boolean_t IA32e = TRUE;
postcode(I386_INIT_ENTRY);
pal_i386_init();
tsc_init();
rtclock_early_init();
kernel_debug_string("i386_init");
pstate_trace();
#if CONFIG_MCA
mca_cpu_init();
#endif
master_cpu = 0;
cpu_init();
postcode(CPU_INIT_D);
printf_init();
panic_init();
kernel_debug_string("PE_init_kprintf");
PE_init_kprintf(FALSE);
kernel_debug_string("kernel_early_bootstrap");
kernel_early_bootstrap();
if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof (dgWork.dgFlags)))
dgWork.dgFlags = 0;
serialmode = 0;
if(PE_parse_boot_argn("serial", &serialmode, sizeof (serialmode))) {
kprintf("Serial mode specified: %08X\n", serialmode);
}
if(serialmode & 1) {
(void)switch_to_serial_console();
disableConsoleOutput = FALSE;
}
kernel_debug_string("PE_init_printf");
PE_init_printf(FALSE);
kprintf("version_variant = %s\n", version_variant);
kprintf("version = %s\n", version);
if (!PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem)))
maxmemtouse = 0;
else
maxmemtouse = ((uint64_t)maxmem) * MB;
if (PE_parse_boot_argn("cpus", &cpus, sizeof (cpus))) {
if ((0 < cpus) && (cpus < max_ncpus))
max_ncpus = cpus;
}
PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof (vm_himemory_mode));
if (vm_himemory_mode != 0)
kprintf("himemory_mode: %d\n", vm_himemory_mode);
if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof (fidn)))
force_immediate_debugger_NMI = FALSE;
else
force_immediate_debugger_NMI = fidn;
#if DEBUG
nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold);
#endif
PE_parse_boot_argn("urgency_notification_abstime",
&urgency_notification_assert_abstime_threshold,
sizeof(urgency_notification_assert_abstime_threshold));
if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
nx_enabled = 0;
kernel_debug_string("i386_vm_init");
i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
PE_init_platform(TRUE, kernelBootArgs);
PE_create_console();
kernel_debug_string("power_management_init");
power_management_init();
processor_bootstrap();
thread_bootstrap();
pstate_trace();
kernel_debug_string("machine_startup");
machine_startup();
pstate_trace();
}
static void
do_init_slave(boolean_t fast_restart)
{
void *init_param = FULL_SLAVE_INIT;
postcode(I386_INIT_SLAVE);
if (!fast_restart) {
set_cr0(get_cr0() & ~(CR0_NW|CR0_CD));
DBG("i386_init_slave() CPU%d: phys (%d) active.\n",
get_cpu_number(), get_cpu_phys_number());
assert(!ml_get_interrupts_enabled());
cpu_mode_init(current_cpu_datap());
pmap_cpu_init();
#if CONFIG_MCA
mca_cpu_init();
#endif
LAPIC_INIT();
lapic_configure();
LAPIC_DUMP();
LAPIC_CPU_MAP_DUMP();
init_fpu();
#if CONFIG_MTRR
mtrr_update_cpu();
#endif
ucode_update_wake();
} else
init_param = FAST_SLAVE_INIT;
#if CONFIG_VMX
vmx_resume();
#endif
#if CONFIG_MTRR
if (!fast_restart)
pat_init();
#endif
cpu_thread_init();
cpu_init();
slave_main(init_param);
panic("do_init_slave() returned from slave_main()");
}
void
i386_init_slave(void)
{
do_init_slave(FALSE);
}
void
i386_init_slave_fast(void)
{
do_init_slave(TRUE);
}