#include <i386/proc_reg.h>
#include <i386/cpuid.h>
#include <i386/tsc.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <i386/pmap_internal.h>
#include <i386/pmap_pcid.h>
#include <mach/branch_predicates.h>
uint32_t pmap_pcid_ncpus;
boolean_t pmap_pcid_disabled = FALSE;
void pmap_pcid_configure(void) {
int ccpu = cpu_number();
uintptr_t cr4 = get_cr4();
boolean_t pcid_present = FALSE;
pmap_pcid_log("PCID configure invoked on CPU %d\n", ccpu);
pmap_assert(ml_get_interrupts_enabled() == FALSE || get_preemption_level() !=0);
pmap_assert(cpu_mode_is64bit());
if (PE_parse_boot_argn("-pmap_pcid_disable", &pmap_pcid_disabled, sizeof (pmap_pcid_disabled))) {
pmap_pcid_log("PMAP: PCID feature disabled\n");
printf("PMAP: PCID feature disabled, %u\n", pmap_pcid_disabled);
kprintf("PMAP: PCID feature disabled %u\n", pmap_pcid_disabled);
}
#if DEBUG
if (pmap_pcid_disabled == FALSE)
no_shared_cr3 = FALSE;
else
no_shared_cr3 = TRUE;
#else
if (no_shared_cr3)
pmap_pcid_disabled = TRUE;
#endif
if (pmap_pcid_disabled || no_shared_cr3) {
unsigned i;
for (i = 0; i < real_ncpus; i++) {
if (cpu_datap(i)) {
cpu_datap(i)->cpu_pmap_pcid_enabled = FALSE;
}
pmap_pcid_ncpus = 0;
}
cpu_datap(ccpu)->cpu_pmap_pcid_enabled = FALSE;
return;
}
if ((cpuid_features() & CPUID_FEATURE_PCID))
pcid_present = TRUE;
else {
cpu_datap(ccpu)->cpu_pmap_pcid_enabled = FALSE;
pmap_pcid_log("PMAP: PCID not detected CPU %d\n", ccpu);
return;
}
if ((cr4 & (CR4_PCIDE | CR4_PGE)) == (CR4_PCIDE|CR4_PGE)) {
cpu_datap(ccpu)->cpu_pmap_pcid_enabled = TRUE;
pmap_pcid_log("PMAP: PCID already enabled %d\n", ccpu);
return;
}
if (pcid_present == TRUE) {
pmap_pcid_log("Pre-PCID:CR0: 0x%lx, CR3: 0x%lx, CR4(CPU %d): 0x%lx\n", get_cr0(), get_cr3_raw(), ccpu, cr4);
if (cpu_number() >= PMAP_PCID_MAX_CPUS) {
panic("PMAP_PCID_MAX_CPUS %d\n", cpu_number());
}
if ((get_cr4() & CR4_PGE) == 0) {
set_cr4(get_cr4() | CR4_PGE);
pmap_pcid_log("Toggled PGE ON (CPU: %d\n", ccpu);
}
set_cr4(get_cr4() | CR4_PCIDE);
pmap_pcid_log("Post PCID: CR0: 0x%lx, CR3: 0x%lx, CR4(CPU %d): 0x%lx\n", get_cr0(), get_cr3_raw(), ccpu, get_cr4());
tlb_flush_global();
cpu_datap(ccpu)->cpu_pmap_pcid_enabled = TRUE;
if (OSIncrementAtomic(&pmap_pcid_ncpus) == machine_info.max_cpus) {
pmap_pcid_log("All PCIDs enabled: real_ncpus: %d, pmap_pcid_ncpus: %d\n", real_ncpus, pmap_pcid_ncpus);
}
cpu_datap(ccpu)->cpu_pmap_pcid_coherentp =
cpu_datap(ccpu)->cpu_pmap_pcid_coherentp_kernel =
&(kernel_pmap->pmap_pcid_coherency_vector[ccpu]);
cpu_datap(ccpu)->cpu_pcid_refcounts[0] = 1;
}
}
void pmap_pcid_initialize(pmap_t p) {
unsigned i;
unsigned nc = sizeof(p->pmap_pcid_cpus)/sizeof(pcid_t);
pmap_assert(nc >= real_ncpus);
for (i = 0; i < nc; i++) {
p->pmap_pcid_cpus[i] = PMAP_PCID_INVALID_PCID;
}
}
void pmap_pcid_initialize_kernel(pmap_t p) {
unsigned i;
unsigned nc = sizeof(p->pmap_pcid_cpus)/sizeof(pcid_t);
for (i = 0; i < nc; i++) {
p->pmap_pcid_cpus[i] = 0;
}
}
pcid_t pmap_pcid_allocate_pcid(int ccpu) {
int i;
pcid_ref_t cur_min = 0xFF;
uint32_t cur_min_index = ~1;
pcid_ref_t *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_refcounts[0];
pcid_ref_t old_count;
if ((i = cpu_datap(ccpu)->cpu_pcid_free_hint) != 0) {
if (cpu_pcid_refcounts[i] == 0) {
(void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
cpu_datap(ccpu)->cpu_pcid_free_hint = 0;
return i;
}
}
for (i = PMAP_PCID_MIN_PCID; i < PMAP_PCID_MAX_PCID; i++) {
pcid_ref_t cur_refcount = cpu_pcid_refcounts[i];
pmap_assert(cur_refcount < PMAP_PCID_MAX_REFCOUNT);
if (cur_refcount == 0) {
(void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
return i;
}
else {
if (cur_refcount < cur_min) {
cur_min_index = i;
cur_min = cur_refcount;
}
}
}
pmap_assert(cur_min_index > 0 && cur_min_index < PMAP_PCID_MAX_PCID);
old_count = __sync_fetch_and_add(&cpu_pcid_refcounts[cur_min_index], 1);
pmap_assert(old_count < PMAP_PCID_MAX_REFCOUNT);
return cur_min_index;
}
void pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) {
pcid_t pcid;
pmap_t lp;
pcid_ref_t prior_count;
pcid = tpmap->pmap_pcid_cpus[ccpu];
pmap_assert(pcid != PMAP_PCID_INVALID_PCID);
if (pcid == PMAP_PCID_INVALID_PCID)
return;
lp = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid];
pmap_assert(pcid > 0 && pcid < PMAP_PCID_MAX_PCID);
pmap_assert(cpu_datap(ccpu)->cpu_pcid_refcounts[pcid] >= 1);
if (lp == tpmap)
(void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID);
if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_refcounts[pcid], 1)) == 1) {
cpu_datap(ccpu)->cpu_pcid_free_hint = pcid;
}
pmap_assert(prior_count <= PMAP_PCID_MAX_REFCOUNT);
}
void pmap_destroy_pcid_sync(pmap_t p) {
int i;
pmap_assert(ml_get_interrupts_enabled() == FALSE || get_preemption_level() !=0);
for (i = 0; i < PMAP_PCID_MAX_CPUS; i++)
if (p->pmap_pcid_cpus[i] != PMAP_PCID_INVALID_PCID)
pmap_pcid_deallocate_pcid(i, p);
}
pcid_t pcid_for_pmap_cpu_tuple(pmap_t pmap, int ccpu) {
return pmap->pmap_pcid_cpus[ccpu];
}
#if PMAP_ASSERT
#define PCID_RECORD_SIZE 128
uint64_t pcid_record_array[PCID_RECORD_SIZE];
#endif
void pmap_pcid_activate(pmap_t tpmap, int ccpu) {
pcid_t new_pcid = tpmap->pmap_pcid_cpus[ccpu];
pmap_t last_pmap;
boolean_t pcid_conflict = FALSE, pending_flush = FALSE;
pmap_assert(cpu_datap(ccpu)->cpu_pmap_pcid_enabled);
if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) {
new_pcid = tpmap->pmap_pcid_cpus[ccpu] = pmap_pcid_allocate_pcid(ccpu);
}
pmap_assert(new_pcid != PMAP_PCID_INVALID_PCID);
#ifdef PCID_ASSERT
cpu_datap(ccpu)->cpu_last_pcid = cpu_datap(ccpu)->cpu_active_pcid;
#endif
cpu_datap(ccpu)->cpu_active_pcid = new_pcid;
pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
if (__probable(pending_flush == FALSE)) {
last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid];
pcid_conflict = ((last_pmap != NULL) &&(tpmap != last_pmap));
}
if (__improbable(pending_flush || pcid_conflict)) {
pmap_pcid_validate_cpu(tpmap, ccpu);
}
cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
pmap_assert(new_pcid < PMAP_PCID_MAX_PCID);
pmap_assert(((tpmap == kernel_pmap) && new_pcid == 0) || ((new_pcid != PMAP_PCID_INVALID_PCID) && (new_pcid != 0)));
#if PMAP_ASSERT
pcid_record_array[ccpu % PCID_RECORD_SIZE] = tpmap->pm_cr3 | new_pcid | (((uint64_t)(!(pending_flush || pcid_conflict))) <<63);
pml4_entry_t *pml4 = pmap64_pml4(tpmap, 0ULL);
if (pml4[KERNEL_PML4_INDEX] != kernel_pmap->pm_pml4[KERNEL_PML4_INDEX])
__asm__ volatile("int3");
#endif
set_cr3_composed(tpmap->pm_cr3, new_pcid, !(pending_flush || pcid_conflict));
if (!pending_flush) {
pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
if (__improbable(pending_flush != 0)) {
pmap_pcid_validate_cpu(tpmap, ccpu);
set_cr3_composed(tpmap->pm_cr3, new_pcid, FALSE);
}
}
cpu_datap(ccpu)->cpu_pmap_pcid_coherentp = &(tpmap->pmap_pcid_coherency_vector[ccpu]);
#if DEBUG
KERNEL_DEBUG_CONSTANT(0x9c1d0000, tpmap, new_pcid, pending_flush, pcid_conflict, 0);
#endif
}