#include <string.h>
#include <norma_vm.h>
#include <mach_kdb.h>
#include <mach_ldebug.h>
#include <mach/machine/vm_types.h>
#include <mach/boolean.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <kern/lock.h>
#include <kern/kalloc.h>
#include <kern/spl.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <mach/vm_param.h>
#include <mach/vm_prot.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <mach/machine/vm_param.h>
#include <machine/thread.h>
#include <kern/misc_protos.h>
#include <i386/misc_protos.h>
#include <i386/cpuid.h>
#include <i386/cpu_data.h>
#include <i386/cpu_number.h>
#include <i386/machine_cpu.h>
#include <i386/mp_slave_boot.h>
#if MACH_KDB
#include <ddb/db_command.h>
#include <ddb/db_output.h>
#include <ddb/db_sym.h>
#include <ddb/db_print.h>
#endif
#include <kern/xpr.h>
#include <vm/vm_protos.h>
#include <i386/mp.h>
void pmap_expand(
pmap_t map,
vm_offset_t v);
extern void pmap_remove_range(
pmap_t pmap,
vm_offset_t va,
pt_entry_t *spte,
pt_entry_t *epte);
void phys_attribute_clear(
ppnum_t phys,
int bits);
boolean_t phys_attribute_test(
ppnum_t phys,
int bits);
void phys_attribute_set(
ppnum_t phys,
int bits);
void pmap_growkernel(
vm_offset_t addr);
void pmap_set_reference(
ppnum_t pn);
void pmap_movepage(
unsigned long from,
unsigned long to,
vm_size_t size);
pt_entry_t * pmap_mapgetpte(
vm_map_t map,
vm_offset_t v);
boolean_t phys_page_exists(
ppnum_t pn);
#ifndef set_dirbase
void set_dirbase(vm_offset_t dirbase);
#endif
#define iswired(pte) ((pte) & INTEL_PTE_WIRED)
#define WRITE_PTE(pte_p, pte_entry) *(pte_p) = (pte_entry);
#define WRITE_PTE_FAST(pte_p, pte_entry) *(pte_p) = (pte_entry);
#define value_64bit(value) ((value) & 0xFFFFFFFF00000000LL)
#define low32(x) ((unsigned int)((x) & 0x00000000ffffffffLL))
typedef struct pv_entry {
struct pv_entry *next;
pmap_t pmap;
vm_offset_t va;
} *pv_entry_t;
#define PV_ENTRY_NULL ((pv_entry_t) 0)
pv_entry_t pv_head_table;
pv_entry_t pv_free_list;
decl_simple_lock_data(,pv_free_list_lock)
int pv_free_count = 0;
#define PV_LOW_WATER_MARK 5000
#define PV_ALLOC_CHUNK 2000
thread_call_t mapping_adjust_call;
static thread_call_data_t mapping_adjust_call_data;
int mappingrecurse = 0;
#define PV_ALLOC(pv_e) { \
simple_lock(&pv_free_list_lock); \
if ((pv_e = pv_free_list) != 0) { \
pv_free_list = pv_e->next; \
pv_free_count--; \
if (pv_free_count < PV_LOW_WATER_MARK) \
if (hw_compare_and_store(0,1,&mappingrecurse)) \
thread_call_enter(mapping_adjust_call); \
} \
simple_unlock(&pv_free_list_lock); \
}
#define PV_FREE(pv_e) { \
simple_lock(&pv_free_list_lock); \
pv_e->next = pv_free_list; \
pv_free_list = pv_e; \
pv_free_count++; \
simple_unlock(&pv_free_list_lock); \
}
zone_t pv_list_zone;
#ifdef PAE
static zone_t pdpt_zone;
#endif
char *pv_lock_table;
#define pv_lock_table_size(n) (((n)+BYTE_SIZE-1)/BYTE_SIZE)
pmap_paddr_t vm_first_phys = (pmap_paddr_t) 0;
pmap_paddr_t vm_last_phys = (pmap_paddr_t) 0;
boolean_t pmap_initialized = FALSE;
pmap_paddr_t kernel_vm_end = (pmap_paddr_t)0;
#define GROW_KERNEL_FUNCTION_IMPLEMENTED 1
#if GROW_KERNEL_FUNCTION_IMPLEMENTED
static struct vm_object kptobj_object_store;
static vm_object_t kptobj;
#endif
#define pa_index(pa) (i386_btop(pa - vm_first_phys))
#define pai_to_pvh(pai) (&pv_head_table[pai])
#define lock_pvh_pai(pai) bit_lock(pai, (void *)pv_lock_table)
#define unlock_pvh_pai(pai) bit_unlock(pai, (void *)pv_lock_table)
char *pmap_phys_attributes;
#define PHYS_MODIFIED INTEL_PTE_MOD
#define PHYS_REFERENCED INTEL_PTE_REF
#define PHYS_NCACHE INTEL_PTE_NCACHE
#define PDE_MAPPED_SIZE (pdetova(1))
#define SPLVM(spl) { \
spl = splhigh(); \
mp_disable_preemption(); \
i_bit_clear(cpu_number(), &cpus_active); \
mp_enable_preemption(); \
}
#define SPLX(spl) { \
mp_disable_preemption(); \
i_bit_set(cpu_number(), &cpus_active); \
mp_enable_preemption(); \
splx(spl); \
}
lock_t pmap_system_lock;
#define PMAP_READ_LOCK(pmap, spl) { \
SPLVM(spl); \
lock_read(&pmap_system_lock); \
simple_lock(&(pmap)->lock); \
}
#define PMAP_WRITE_LOCK(spl) { \
SPLVM(spl); \
lock_write(&pmap_system_lock); \
}
#define PMAP_READ_UNLOCK(pmap, spl) { \
simple_unlock(&(pmap)->lock); \
lock_read_done(&pmap_system_lock); \
SPLX(spl); \
}
#define PMAP_WRITE_UNLOCK(spl) { \
lock_write_done(&pmap_system_lock); \
SPLX(spl); \
}
#define PMAP_WRITE_TO_READ_LOCK(pmap) { \
simple_lock(&(pmap)->lock); \
lock_write_to_read(&pmap_system_lock); \
}
#define LOCK_PVH(index) lock_pvh_pai(index)
#define UNLOCK_PVH(index) unlock_pvh_pai(index)
#if USLOCK_DEBUG
extern int max_lock_loops;
extern int disableSerialOuput;
#define LOOP_VAR \
unsigned int loop_count; \
loop_count = disableSerialOuput ? max_lock_loops \
: max_lock_loops*100
#define LOOP_CHECK(msg, pmap) \
if (--loop_count == 0) { \
mp_disable_preemption(); \
kprintf("%s: cpu %d pmap %x, cpus_active 0x%x\n", \
msg, cpu_number(), pmap, cpus_active); \
Debugger("deadlock detection"); \
mp_enable_preemption(); \
loop_count = max_lock_loops; \
}
#else
#define LOOP_VAR
#define LOOP_CHECK(msg, pmap)
#endif
#define PMAP_UPDATE_TLBS(pmap, s, e) \
{ \
cpu_set cpu_mask; \
cpu_set users; \
\
mp_disable_preemption(); \
cpu_mask = 1 << cpu_number(); \
\
\
\
\
\
users = (pmap)->cpus_using & ~cpu_mask; \
if (users) { \
LOOP_VAR; \
\
\
signal_cpus(users, (pmap), (s), (e)); \
while (((pmap)->cpus_using & cpus_active & ~cpu_mask)) { \
LOOP_CHECK("PMAP_UPDATE_TLBS", pmap); \
cpu_pause(); \
} \
} \
\
\
if ((pmap)->cpus_using & cpu_mask) { \
INVALIDATE_TLB((pmap), (s), (e)); \
} \
\
mp_enable_preemption(); \
}
#define MAX_TBIS_SIZE 32
#define INVALIDATE_TLB(m, s, e) { \
flush_tlb(); \
}
cpu_set cpus_active;
cpu_set cpus_idle;
#define UPDATE_LIST_SIZE 4
struct pmap_update_item {
pmap_t pmap;
vm_offset_t start;
vm_offset_t end;
};
typedef struct pmap_update_item *pmap_update_item_t;
struct pmap_update_list {
decl_simple_lock_data(,lock)
int count;
struct pmap_update_item item[UPDATE_LIST_SIZE];
} ;
typedef struct pmap_update_list *pmap_update_list_t;
extern void signal_cpus(
cpu_set use_list,
pmap_t pmap,
vm_offset_t start,
vm_offset_t end);
pmap_memory_region_t pmap_memory_regions[PMAP_MEMORY_REGIONS_SIZE];
#define current_pmap() (vm_map_pmap(current_thread()->map))
#define pmap_in_use(pmap, cpu) (((pmap)->cpus_using & (1 << (cpu))) != 0)
struct pmap kernel_pmap_store;
pmap_t kernel_pmap;
#ifdef PMAP_QUEUE
decl_simple_lock_data(,free_pmap_lock)
#endif
struct zone *pmap_zone;
int pmap_debug = 0;
unsigned int inuse_ptepages_count = 0;
int pmap_cache_max = 32;
int pmap_alloc_chunk = 8;
pmap_t pmap_cache_list;
int pmap_cache_count;
decl_simple_lock_data(,pmap_cache_lock)
extern vm_offset_t hole_start, hole_end;
extern char end;
static int nkpt;
pt_entry_t *DMAP1, *DMAP2;
caddr_t DADDR1;
caddr_t DADDR2;
#if DEBUG_ALIAS
#define PMAP_ALIAS_MAX 32
struct pmap_alias {
vm_offset_t rpc;
pmap_t pmap;
vm_offset_t va;
int cookie;
#define PMAP_ALIAS_COOKIE 0xdeadbeef
} pmap_aliasbuf[PMAP_ALIAS_MAX];
int pmap_alias_index = 0;
extern vm_offset_t get_rpc();
#endif
#define pmap_pde(m, v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))
#define pdir_pde(d, v) (d[(vm_offset_t)(v) >> PDESHIFT])
static __inline int
pmap_is_current(pmap_t pmap)
{
return (pmap == kernel_pmap ||
(pmap->dirbase[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME));
}
pt_entry_t *
pmap_pte(pmap_t pmap, vm_offset_t va)
{
pd_entry_t *pde;
pd_entry_t newpf;
pde = pmap_pde(pmap, va);
if (*pde != 0) {
if (pmap_is_current(pmap))
return( vtopte(va));
newpf = *pde & PG_FRAME;
if (((*CM4) & PG_FRAME) != newpf) {
*CM4 = newpf | INTEL_PTE_RW | INTEL_PTE_VALID;
invlpg((u_int)CA4);
}
return (pt_entry_t *)CA4 + (i386_btop(va) & (NPTEPG-1));
}
return(0);
}
#define DEBUG_PTE_PAGE 0
#if DEBUG_PTE_PAGE
void
ptep_check(
ptep_t ptep)
{
register pt_entry_t *pte, *epte;
int ctu, ctw;
if (ptep == PTE_PAGE_NULL)
return;
pte = pmap_pte(ptep->pmap, ptep->va);
epte = pte + INTEL_PGBYTES/sizeof(pt_entry_t);
ctu = 0;
ctw = 0;
while (pte < epte) {
if (pte->pfn != 0) {
ctu++;
if (pte->wired)
ctw++;
}
pte++;
}
if (ctu != ptep->use_count || ctw != ptep->wired_count) {
printf("use %d wired %d - actual use %d wired %d\n",
ptep->use_count, ptep->wired_count, ctu, ctw);
panic("pte count");
}
}
#endif
vm_offset_t
pmap_map(
register vm_offset_t virt,
register vm_offset_t start_addr,
register vm_offset_t end_addr,
register vm_prot_t prot)
{
register int ps;
ps = PAGE_SIZE;
while (start_addr < end_addr) {
pmap_enter(kernel_pmap,
virt, (ppnum_t) i386_btop(start_addr), prot, 0, FALSE);
virt += ps;
start_addr += ps;
}
return(virt);
}
vm_offset_t
pmap_map_bd(
register vm_offset_t virt,
register vm_offset_t start_addr,
register vm_offset_t end_addr,
vm_prot_t prot)
{
register pt_entry_t template;
register pt_entry_t *pte;
template = pa_to_pte(start_addr)
| INTEL_PTE_NCACHE
| INTEL_PTE_REF
| INTEL_PTE_MOD
| INTEL_PTE_WIRED
| INTEL_PTE_VALID;
if (prot & VM_PROT_WRITE)
template |= INTEL_PTE_WRITE;
while (start_addr < end_addr) {
pte = pmap_pte(kernel_pmap, virt);
if (pte == PT_ENTRY_NULL) {
panic("pmap_map_bd: Invalid kernel address\n");
}
WRITE_PTE_FAST(pte, template)
pte_increment_pa(template);
virt += PAGE_SIZE;
start_addr += PAGE_SIZE;
}
flush_tlb();
return(virt);
}
extern char *first_avail;
extern vm_offset_t virtual_avail, virtual_end;
extern pmap_paddr_t avail_start, avail_end;
extern vm_offset_t etext;
extern void *sectHIBB;
extern int sectSizeHIB;
void
pmap_bootstrap(
__unused vm_offset_t load_start)
{
vm_offset_t va;
pt_entry_t *pte;
int i;
int wpkernel, boot_arg;
vm_last_addr = VM_MAX_KERNEL_ADDRESS;
kernel_pmap = &kernel_pmap_store;
#ifdef PMAP_QUEUE
kernel_pmap->pmap_link.next = (queue_t)kernel_pmap;
kernel_pmap->pmap_link.prev = (queue_t)kernel_pmap;
#endif
kernel_pmap->ref_count = 1;
kernel_pmap->pm_obj = (vm_object_t) NULL;
kernel_pmap->dirbase = (pd_entry_t *)((unsigned int)IdlePTD | KERNBASE);
kernel_pmap->pdirbase = (pd_entry_t *)IdlePTD;
#ifdef PAE
kernel_pmap->pm_pdpt = (pd_entry_t *)((unsigned int)IdlePDPT | KERNBASE );
kernel_pmap->pm_ppdpt = (vm_offset_t)IdlePDPT;
#endif
va = (vm_offset_t)kernel_pmap->dirbase;
for (i = 0; i< NPGPTD; i++ ) {
pmap_paddr_t pa;
pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
* (pd_entry_t *) (kernel_pmap->dirbase + PTDPTDI + i) =
(pa & PG_FRAME) | INTEL_PTE_VALID | INTEL_PTE_RW | INTEL_PTE_REF |
INTEL_PTE_MOD | INTEL_PTE_WIRED ;
#ifdef PAE
kernel_pmap->pm_pdpt[i] = pa | INTEL_PTE_VALID;
#endif
}
nkpt = NKPT;
virtual_avail = (vm_offset_t)VADDR(KPTDI,0) + (vm_offset_t)first_avail;
virtual_end = (vm_offset_t)(VM_MAX_KERNEL_ADDRESS);
#define SYSMAP(c, p, v, n) \
v = (c)va; va += ((n)*INTEL_PGBYTES); p = pte; pte += (n);
va = virtual_avail;
pte = (pt_entry_t *) pmap_pte(kernel_pmap, va);
SYSMAP(caddr_t, CM1, CA1, 1)
* (pt_entry_t *) CM1 = 0;
SYSMAP(caddr_t, CM2, CA2, 1)
* (pt_entry_t *) CM2 = 0;
SYSMAP(caddr_t, CM3, CA3, 1)
* (pt_entry_t *) CM3 = 0;
SYSMAP(caddr_t, CM4, CA4, 1)
* (pt_entry_t *) CM4 = 0;
SYSMAP(caddr_t, DMAP1, DADDR1, 1);
SYSMAP(caddr_t, DMAP2, DADDR2, 1);
lock_init(&pmap_system_lock,
FALSE,
0, 0);
virtual_avail = va;
wpkernel = 1;
if (PE_parse_boot_arg("debug", &boot_arg)) {
if (boot_arg & DB_PRT) wpkernel = 0;
if (boot_arg & DB_NMI) wpkernel = 0;
}
if (wpkernel)
{
vm_offset_t myva;
pt_entry_t *ptep;
for (myva = i386_round_page(VM_MIN_KERNEL_ADDRESS + MP_BOOT + MP_BOOTSTACK); myva < etext; myva += PAGE_SIZE) {
if (myva >= (vm_offset_t)sectHIBB && myva < ((vm_offset_t)sectHIBB + sectSizeHIB))
continue;
ptep = pmap_pte(kernel_pmap, myva);
if (ptep)
*ptep &= ~INTEL_PTE_RW;
}
flush_tlb();
}
simple_lock_init(&kernel_pmap->lock, 0);
simple_lock_init(&pv_free_list_lock, 0);
memset((char *)kernel_pmap->dirbase,
0,
(KPTDI) * sizeof(pd_entry_t));
kprintf("Kernel virtual space from 0x%x to 0x%x.\n",
VADDR(KPTDI,0), virtual_end);
#ifdef PAE
kprintf("Available physical space from 0x%llx to 0x%llx\n",
avail_start, avail_end);
printf("PAE enabled\n");
#else
kprintf("Available physical space from 0x%x to 0x%x\n",
avail_start, avail_end);
#endif
}
void
pmap_virtual_space(
vm_offset_t *startp,
vm_offset_t *endp)
{
*startp = virtual_avail;
*endp = virtual_end;
}
void
pmap_init(void)
{
register long npages;
vm_offset_t addr;
register vm_size_t s;
vm_offset_t vaddr;
ppnum_t ppn;
npages = i386_btop(avail_end);
s = (vm_size_t) (sizeof(struct pv_entry) * npages
+ pv_lock_table_size(npages)
+ npages);
s = round_page(s);
if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS)
panic("pmap_init");
memset((char *)addr, 0, s);
pv_head_table = (pv_entry_t) addr;
addr = (vm_offset_t) (pv_head_table + npages);
pv_lock_table = (char *) addr;
addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
pmap_phys_attributes = (char *) addr;
s = (vm_size_t) sizeof(struct pmap);
pmap_zone = zinit(s, 400*s, 4096, "pmap");
s = (vm_size_t) sizeof(struct pv_entry);
pv_list_zone = zinit(s, 10000*s, 4096, "pv_list");
#ifdef PAE
s = 63;
pdpt_zone = zinit(s, 400*s, 4096, "pdpt");
#endif
vm_first_phys = 0;
vm_last_phys = avail_end;
#if GROW_KERNEL_FUNCTION_IMPLEMENTED
kptobj = &kptobj_object_store;
_vm_object_allocate((vm_object_size_t)NKPDE, kptobj);
kernel_pmap->pm_obj = kptobj;
#endif
vaddr = (vm_offset_t)VM_MIN_KERNEL_ADDRESS;
for (ppn = 0; ppn < i386_btop(avail_start) ; ppn++ ) {
pv_entry_t pv_e;
pv_e = pai_to_pvh(ppn);
pv_e->va = vaddr;
vaddr += PAGE_SIZE;
pv_e->pmap = kernel_pmap;
pv_e->next = PV_ENTRY_NULL;
}
pmap_initialized = TRUE;
pmap_cache_list = PMAP_NULL;
pmap_cache_count = 0;
simple_lock_init(&pmap_cache_lock, 0);
#ifdef PMAP_QUEUE
simple_lock_init(&free_pmap_lock, 0);
#endif
}
void
x86_lowmem_free(void)
{
ml_static_mfree((vm_offset_t) i386_ptob(pmap_memory_regions[0].base)|VM_MIN_KERNEL_ADDRESS,
(vm_size_t) i386_ptob(pmap_memory_regions[0].end - pmap_memory_regions[0].base));
}
#define valid_page(x) (pmap_initialized && pmap_valid_page(x))
boolean_t
pmap_verify_free(
ppnum_t pn)
{
pmap_paddr_t phys;
pv_entry_t pv_h;
int pai;
spl_t spl;
boolean_t result;
assert(pn != vm_page_fictitious_addr);
phys = (pmap_paddr_t)i386_ptob(pn);
if (!pmap_initialized)
return(TRUE);
if (!pmap_valid_page(pn))
return(FALSE);
PMAP_WRITE_LOCK(spl);
pai = pa_index(phys);
pv_h = pai_to_pvh(pai);
result = (pv_h->pmap == PMAP_NULL);
PMAP_WRITE_UNLOCK(spl);
return(result);
}
pmap_t
pmap_create(
vm_size_t size)
{
register pmap_t p;
#ifdef PMAP_QUEUE
register pmap_t pro;
spl_t s;
#endif
register int i;
register vm_offset_t va;
if (size != 0) {
return(PMAP_NULL);
}
p = (pmap_t) zalloc(pmap_zone);
if (PMAP_NULL == p)
panic("pmap_create zalloc");
if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->dirbase), NBPTD))
panic("pmap_create kmem_alloc_wired");
#ifdef PAE
p->pm_hold = (vm_offset_t)zalloc(pdpt_zone);
if ((vm_offset_t)NULL == p->pm_hold) {
panic("pdpt zalloc");
}
p->pm_pdpt = (pdpt_entry_t *) (( p->pm_hold + 31) & ~31);
p->pm_ppdpt = kvtophys((vm_offset_t)p->pm_pdpt);
#endif
if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPGPTD*NPDEPG))))
panic("pmap_create vm_object_allocate");
memcpy(p->dirbase,
(void *)((unsigned int)IdlePTD | KERNBASE),
NBPTD);
va = (vm_offset_t)p->dirbase;
p->pdirbase = (pd_entry_t *)(kvtophys(va));
simple_lock_init(&p->lock, 0);
for (i = 0; i< NPGPTD; i++ ) {
pmap_paddr_t pa;
pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
* (pd_entry_t *) (p->dirbase + PTDPTDI + i) =
(pa & PG_FRAME) | INTEL_PTE_VALID | INTEL_PTE_RW | INTEL_PTE_REF |
INTEL_PTE_MOD | INTEL_PTE_WIRED ;
#ifdef PAE
p->pm_pdpt[i] = pa | INTEL_PTE_VALID;
#endif
}
p->cpus_using = 0;
p->stats.resident_count = 0;
p->stats.wired_count = 0;
p->ref_count = 1;
#ifdef PMAP_QUEUE
SPLVM(s);
simple_lock(&free_pmap_lock);
p->pmap_link.next = (queue_t)kernel_pmap->pmap_link.next;
kernel_pmap->pmap_link.next = (queue_t)p;
pro = (pmap_t) p->pmap_link.next;
p->pmap_link.prev = (queue_t)pro->pmap_link.prev;
pro->pmap_link.prev = (queue_t)p;
simple_unlock(&free_pmap_lock);
SPLX(s);
#endif
return(p);
}
void
pmap_destroy(
register pmap_t p)
{
register pt_entry_t *pdep;
register int c;
spl_t s;
register vm_page_t m;
#ifdef PMAP_QUEUE
register pmap_t pre,pro;
#endif
if (p == PMAP_NULL)
return;
SPLVM(s);
simple_lock(&p->lock);
c = --p->ref_count;
if (c == 0) {
register int my_cpu;
mp_disable_preemption();
my_cpu = cpu_number();
PMAP_UPDATE_TLBS(p,
VM_MIN_ADDRESS,
VM_MAX_KERNEL_ADDRESS);
if (PMAP_REAL(my_cpu) == p) {
PMAP_CPU_CLR(p, my_cpu);
PMAP_REAL(my_cpu) = kernel_pmap;
#ifdef PAE
set_cr3((unsigned int)kernel_pmap->pm_ppdpt);
#else
set_cr3((unsigned int)kernel_pmap->pdirbase);
#endif
}
mp_enable_preemption();
}
simple_unlock(&p->lock);
SPLX(s);
if (c != 0) {
return;
}
#ifdef PMAP_QUEUE
SPLVM(s);
simple_lock(&free_pmap_lock);
pre = (pmap_t)p->pmap_link.prev;
pre->pmap_link.next = (queue_t)p->pmap_link.next;
pro = (pmap_t)p->pmap_link.next;
pro->pmap_link.prev = (queue_t)p->pmap_link.prev;
simple_unlock(&free_pmap_lock);
SPLX(s);
#endif
pdep = (pt_entry_t *)p->dirbase;
while (pdep < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)]) {
int ind;
if (*pdep & INTEL_PTE_VALID) {
ind = pdep - (pt_entry_t *)&p->dirbase[0];
vm_object_lock(p->pm_obj);
m = vm_page_lookup(p->pm_obj, (vm_object_offset_t)ind);
if (m == VM_PAGE_NULL) {
panic("pmap_destroy: pte page not in object");
}
vm_page_lock_queues();
vm_page_free(m);
inuse_ptepages_count--;
vm_object_unlock(p->pm_obj);
vm_page_unlock_queues();
*pdep++ = 0;
}
else {
*pdep++ = 0;
}
}
vm_object_deallocate(p->pm_obj);
kmem_free(kernel_map, (vm_offset_t)p->dirbase, NBPTD);
#ifdef PAE
zfree(pdpt_zone, (void *)p->pm_hold);
#endif
zfree(pmap_zone, p);
}
void
pmap_reference(
register pmap_t p)
{
spl_t s;
if (p != PMAP_NULL) {
SPLVM(s);
simple_lock(&p->lock);
p->ref_count++;
simple_unlock(&p->lock);
SPLX(s);
}
}
void
pmap_remove_range(
pmap_t pmap,
vm_offset_t va,
pt_entry_t *spte,
pt_entry_t *epte)
{
register pt_entry_t *cpte;
int num_removed, num_unwired;
int pai;
pmap_paddr_t pa;
#if DEBUG_PTE_PAGE
if (pmap != kernel_pmap)
ptep_check(get_pte_page(spte));
#endif
num_removed = 0;
num_unwired = 0;
for (cpte = spte; cpte < epte;
cpte++, va += PAGE_SIZE) {
pa = pte_to_pa(*cpte);
if (pa == 0)
continue;
num_removed++;
if (iswired(*cpte))
num_unwired++;
if (!valid_page(i386_btop(pa))) {
register pt_entry_t *lpte = cpte;
*lpte = 0;
continue;
}
pai = pa_index(pa);
LOCK_PVH(pai);
{
register pt_entry_t *lpte;
lpte = cpte;
pmap_phys_attributes[pai] |=
*lpte & (PHYS_MODIFIED|PHYS_REFERENCED);
*lpte = 0;
}
{
register pv_entry_t pv_h, prev, cur;
pv_h = pai_to_pvh(pai);
if (pv_h->pmap == PMAP_NULL) {
panic("pmap_remove: null pv_list!");
}
if (pv_h->va == va && pv_h->pmap == pmap) {
cur = pv_h->next;
if (cur != PV_ENTRY_NULL) {
*pv_h = *cur;
PV_FREE(cur);
}
else {
pv_h->pmap = PMAP_NULL;
}
}
else {
cur = pv_h;
do {
prev = cur;
if ((cur = prev->next) == PV_ENTRY_NULL) {
panic("pmap-remove: mapping not in pv_list!");
}
} while (cur->va != va || cur->pmap != pmap);
prev->next = cur->next;
PV_FREE(cur);
}
UNLOCK_PVH(pai);
}
}
assert(pmap->stats.resident_count >= num_removed);
pmap->stats.resident_count -= num_removed;
assert(pmap->stats.wired_count >= num_unwired);
pmap->stats.wired_count -= num_unwired;
}
void
pmap_remove_some_phys(
__unused pmap_t map,
__unused ppnum_t pn)
{
}
void
pmap_remove(
pmap_t map,
addr64_t s64,
addr64_t e64)
{
spl_t spl;
register pt_entry_t *pde;
register pt_entry_t *spte, *epte;
vm_offset_t l;
vm_offset_t s, e;
vm_offset_t orig_s;
if (map == PMAP_NULL)
return;
PMAP_READ_LOCK(map, spl);
if (value_64bit(s64) || value_64bit(e64)) {
panic("pmap_remove addr overflow");
}
orig_s = s = (vm_offset_t)low32(s64);
e = (vm_offset_t)low32(e64);
pde = pmap_pde(map, s);
while (s < e) {
l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
if (l > e)
l = e;
if (*pde & INTEL_PTE_VALID) {
spte = (pt_entry_t *)pmap_pte(map, (s & ~(PDE_MAPPED_SIZE-1)));
spte = &spte[ptenum(s)];
epte = &spte[intel_btop(l-s)];
pmap_remove_range(map, s, spte, epte);
}
s = l;
pde++;
}
PMAP_UPDATE_TLBS(map, orig_s, e);
PMAP_READ_UNLOCK(map, spl);
}
void
pmap_page_protect(
ppnum_t pn,
vm_prot_t prot)
{
pv_entry_t pv_h, prev;
register pv_entry_t pv_e;
register pt_entry_t *pte;
int pai;
register pmap_t pmap;
spl_t spl;
boolean_t remove;
pmap_paddr_t phys;
assert(pn != vm_page_fictitious_addr);
phys = (pmap_paddr_t)i386_ptob(pn);
if (!valid_page(pn)) {
return;
}
switch (prot) {
case VM_PROT_READ:
case VM_PROT_READ|VM_PROT_EXECUTE:
remove = FALSE;
break;
case VM_PROT_ALL:
return;
default:
remove = TRUE;
break;
}
PMAP_WRITE_LOCK(spl);
pai = pa_index(phys);
pv_h = pai_to_pvh(pai);
if (pv_h->pmap != PMAP_NULL) {
prev = pv_e = pv_h;
do {
register vm_offset_t va;
pmap = pv_e->pmap;
simple_lock(&pmap->lock);
{
va = pv_e->va;
pte = pmap_pte(pmap, va);
}
if (remove || pmap == kernel_pmap) {
{
pmap_phys_attributes[pai] |=
*pte & (PHYS_MODIFIED|PHYS_REFERENCED);
*pte++ = 0;
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
}
assert(pmap->stats.resident_count >= 1);
pmap->stats.resident_count--;
if (pv_e == pv_h) {
pv_h->pmap = PMAP_NULL;
}
else {
prev->next = pv_e->next;
PV_FREE(pv_e);
}
}
else {
*pte &= ~INTEL_PTE_WRITE;
pte++;
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
prev = pv_e;
}
simple_unlock(&pmap->lock);
} while ((pv_e = prev->next) != PV_ENTRY_NULL);
if (pv_h->pmap == PMAP_NULL) {
pv_e = pv_h->next;
if (pv_e != PV_ENTRY_NULL) {
*pv_h = *pv_e;
PV_FREE(pv_e);
}
}
}
PMAP_WRITE_UNLOCK(spl);
}
unsigned int pmap_disconnect(
ppnum_t pa)
{
pmap_page_protect(pa, 0);
return (pmap_get_refmod(pa));
}
void
pmap_protect(
pmap_t map,
vm_offset_t s,
vm_offset_t e,
vm_prot_t prot)
{
register pt_entry_t *pde;
register pt_entry_t *spte, *epte;
vm_offset_t l;
spl_t spl;
vm_offset_t orig_s = s;
if (map == PMAP_NULL)
return;
switch (prot) {
case VM_PROT_READ:
case VM_PROT_READ|VM_PROT_EXECUTE:
break;
case VM_PROT_READ|VM_PROT_WRITE:
case VM_PROT_ALL:
return;
default:
pmap_remove(map, (addr64_t)s, (addr64_t)e);
return;
}
SPLVM(spl);
simple_lock(&map->lock);
pde = pmap_pde(map, s);
while (s < e) {
l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
if (l > e)
l = e;
if (*pde & INTEL_PTE_VALID) {
spte = (pt_entry_t *)pmap_pte(map, (s & ~(PDE_MAPPED_SIZE-1)));
spte = &spte[ptenum(s)];
epte = &spte[intel_btop(l-s)];
while (spte < epte) {
if (*spte & INTEL_PTE_VALID)
*spte &= ~INTEL_PTE_WRITE;
spte++;
}
}
s = l;
pde++;
}
PMAP_UPDATE_TLBS(map, orig_s, e);
simple_unlock(&map->lock);
SPLX(spl);
}
void
pmap_enter(
register pmap_t pmap,
vm_offset_t v,
ppnum_t pn,
vm_prot_t prot,
unsigned int flags,
boolean_t wired)
{
register pt_entry_t *pte;
register pv_entry_t pv_h;
register int pai;
pv_entry_t pv_e;
pt_entry_t template;
spl_t spl;
pmap_paddr_t old_pa;
pmap_paddr_t pa = (pmap_paddr_t)i386_ptob(pn);
XPR(0x80000000, "%x/%x: pmap_enter %x/%x/%x\n",
current_thread(),
current_thread(),
pmap, v, pn);
assert(pn != vm_page_fictitious_addr);
if (pmap_debug)
printf("pmap(%x, %x)\n", v, pn);
if (pmap == PMAP_NULL)
return;
pv_e = PV_ENTRY_NULL;
PMAP_READ_LOCK(pmap, spl);
while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
PMAP_READ_UNLOCK(pmap, spl);
pmap_expand(pmap, v);
PMAP_READ_LOCK(pmap, spl);
}
old_pa = pte_to_pa(*pte);
if (old_pa == pa) {
template = pa_to_pte(pa) | INTEL_PTE_VALID;
if(flags & VM_MEM_NOT_CACHEABLE) {
if(!(flags & VM_MEM_GUARDED))
template |= INTEL_PTE_PTA;
template |= INTEL_PTE_NCACHE;
}
if (pmap != kernel_pmap)
template |= INTEL_PTE_USER;
if (prot & VM_PROT_WRITE)
template |= INTEL_PTE_WRITE;
if (wired) {
template |= INTEL_PTE_WIRED;
if (!iswired(*pte))
pmap->stats.wired_count++;
}
else {
if (iswired(*pte)) {
assert(pmap->stats.wired_count >= 1);
pmap->stats.wired_count--;
}
}
if (*pte & INTEL_PTE_MOD)
template |= INTEL_PTE_MOD;
WRITE_PTE(pte, template)
pte++;
goto Done;
}
if (old_pa != (pmap_paddr_t) 0) {
#if DEBUG_PTE_PAGE
if (pmap != kernel_pmap)
ptep_check(get_pte_page(pte));
#endif
if (valid_page(i386_btop(old_pa))) {
pai = pa_index(old_pa);
LOCK_PVH(pai);
assert(pmap->stats.resident_count >= 1);
pmap->stats.resident_count--;
if (iswired(*pte)) {
assert(pmap->stats.wired_count >= 1);
pmap->stats.wired_count--;
}
pmap_phys_attributes[pai] |=
*pte & (PHYS_MODIFIED|PHYS_REFERENCED);
WRITE_PTE(pte, 0)
{
register pv_entry_t prev, cur;
pv_h = pai_to_pvh(pai);
if (pv_h->pmap == PMAP_NULL) {
panic("pmap_enter: null pv_list!");
}
if (pv_h->va == v && pv_h->pmap == pmap) {
cur = pv_h->next;
if (cur != PV_ENTRY_NULL) {
*pv_h = *cur;
pv_e = cur;
}
else {
pv_h->pmap = PMAP_NULL;
}
}
else {
cur = pv_h;
do {
prev = cur;
if ((cur = prev->next) == PV_ENTRY_NULL) {
panic("pmap_enter: mapping not in pv_list!");
}
} while (cur->va != v || cur->pmap != pmap);
prev->next = cur->next;
pv_e = cur;
}
}
UNLOCK_PVH(pai);
}
else {
old_pa = (pmap_paddr_t) 0;
assert(pmap->stats.resident_count >= 1);
pmap->stats.resident_count--;
if (iswired(*pte)) {
assert(pmap->stats.wired_count >= 1);
pmap->stats.wired_count--;
}
}
}
if (valid_page(i386_btop(pa))) {
pai = pa_index(pa);
#if SHARING_FAULTS
RetryPvList:
#endif
LOCK_PVH(pai);
pv_h = pai_to_pvh(pai);
if (pv_h->pmap == PMAP_NULL) {
pv_h->va = v;
pv_h->pmap = pmap;
pv_h->next = PV_ENTRY_NULL;
}
else {
#if DEBUG
{
pv_entry_t e = pv_h;
while (e != PV_ENTRY_NULL) {
if (e->pmap == pmap && e->va == v)
panic("pmap_enter: already in pv_list");
e = e->next;
}
}
#endif
#if SHARING_FAULTS
{
pv_entry_t e = pv_h;
pt_entry_t *opte;
while (e != PV_ENTRY_NULL) {
if (e->pmap == pmap) {
UNLOCK_PVH(pai);
opte = pmap_pte(pmap, e->va);
assert(opte != PT_ENTRY_NULL);
pmap_remove_range(pmap, e->va, opte,
opte + 1);
PMAP_UPDATE_TLBS(pmap, e->va, e->va + PAGE_SIZE);
goto RetryPvList;
}
e = e->next;
}
e = pv_h;
while (e != PV_ENTRY_NULL) {
if (e->pmap == pmap)
panic("pmap_enter: alias in pv_list");
e = e->next;
}
}
#endif
#if DEBUG_ALIAS
{
pv_entry_t e = pv_h;
vm_offset_t rpc = get_rpc();
while (e != PV_ENTRY_NULL) {
if (e->pmap == pmap) {
struct pmap_alias *pma;
int ii, logit;
logit = TRUE;
for (ii = 0; ii < pmap_alias_index; ii++) {
if (pmap_aliasbuf[ii].rpc == rpc) {
logit = FALSE;
break;
}
}
if (logit) {
pma = &pmap_aliasbuf[pmap_alias_index];
pma->pmap = pmap;
pma->va = v;
pma->rpc = rpc;
pma->cookie = PMAP_ALIAS_COOKIE;
if (++pmap_alias_index >= PMAP_ALIAS_MAX)
panic("pmap_enter: exhausted alias log");
}
}
e = e->next;
}
}
#endif
if (pv_e == PV_ENTRY_NULL) {
PV_ALLOC(pv_e);
if (pv_e == PV_ENTRY_NULL) {
panic("pmap no pv_e's");
}
}
pv_e->va = v;
pv_e->pmap = pmap;
pv_e->next = pv_h->next;
pv_h->next = pv_e;
pv_e = PV_ENTRY_NULL;
}
UNLOCK_PVH(pai);
}
pmap->stats.resident_count++;
template = pa_to_pte(pa) | INTEL_PTE_VALID;
if(flags & VM_MEM_NOT_CACHEABLE) {
if(!(flags & VM_MEM_GUARDED))
template |= INTEL_PTE_PTA;
template |= INTEL_PTE_NCACHE;
}
if (pmap != kernel_pmap)
template |= INTEL_PTE_USER;
if (prot & VM_PROT_WRITE)
template |= INTEL_PTE_WRITE;
if (wired) {
template |= INTEL_PTE_WIRED;
pmap->stats.wired_count++;
}
WRITE_PTE(pte, template)
Done:
PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
if (pv_e != PV_ENTRY_NULL) {
PV_FREE(pv_e);
}
PMAP_READ_UNLOCK(pmap, spl);
}
void
pmap_change_wiring(
register pmap_t map,
vm_offset_t v,
boolean_t wired)
{
register pt_entry_t *pte;
spl_t spl;
#if 1
PMAP_READ_LOCK(map, spl);
if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL)
panic("pmap_change_wiring: pte missing");
if (wired && !iswired(*pte)) {
map->stats.wired_count++;
*pte++ |= INTEL_PTE_WIRED;
}
else if (!wired && iswired(*pte)) {
assert(map->stats.wired_count >= 1);
map->stats.wired_count--;
*pte++ &= ~INTEL_PTE_WIRED;
}
PMAP_READ_UNLOCK(map, spl);
#else
return;
#endif
}
ppnum_t
pmap_find_phys(pmap_t pmap, addr64_t va)
{
pt_entry_t *ptp;
vm_offset_t a32;
ppnum_t ppn;
if (value_64bit(va))
panic("pmap_find_phys 64 bit value");
a32 = (vm_offset_t) low32(va);
ptp = pmap_pte(pmap, a32);
if (PT_ENTRY_NULL == ptp) {
ppn = 0;
} else {
ppn = (ppnum_t) i386_btop(pte_to_pa(*ptp));
}
return ppn;
}
vm_offset_t
pmap_extract(
register pmap_t pmap,
vm_offset_t va)
{
ppnum_t ppn;
vm_offset_t vaddr;
vaddr = (vm_offset_t)0;
ppn = pmap_find_phys(pmap, (addr64_t)va);
if (ppn) {
vaddr = ((vm_offset_t)i386_ptob(ppn)) | (va & INTEL_OFFMASK);
}
return (vaddr);
}
void
pmap_expand(
register pmap_t map,
register vm_offset_t v)
{
pt_entry_t *pdp;
register vm_page_t m;
register pmap_paddr_t pa;
register int i;
spl_t spl;
ppnum_t pn;
if (map == kernel_pmap) {
pmap_growkernel(v);
return;
}
while ((m = vm_page_grab()) == VM_PAGE_NULL)
VM_PAGE_WAIT();
pn = m->phys_page;
pa = i386_ptob(pn);
i = pdenum(map, v);
vm_object_lock(map->pm_obj);
vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i);
vm_page_lock_queues();
vm_page_wire(m);
inuse_ptepages_count++;
vm_object_unlock(map->pm_obj);
vm_page_unlock_queues();
pmap_zero_page(pn);
PMAP_READ_LOCK(map, spl);
if (pmap_pte(map, v) != PT_ENTRY_NULL) {
PMAP_READ_UNLOCK(map, spl);
vm_object_lock(map->pm_obj);
vm_page_lock_queues();
vm_page_free(m);
inuse_ptepages_count--;
vm_page_unlock_queues();
vm_object_unlock(map->pm_obj);
return;
}
pdp = &map->dirbase[pdenum(map, v)];
*pdp = pa_to_pte(pa)
| INTEL_PTE_VALID
| INTEL_PTE_USER
| INTEL_PTE_WRITE;
PMAP_READ_UNLOCK(map, spl);
return;
}
#if 0
void
pmap_copy(
pmap_t dst_pmap,
pmap_t src_pmap,
vm_offset_t dst_addr,
vm_size_t len,
vm_offset_t src_addr)
{
#ifdef lint
dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
#endif
}
#endif/* 0 */
void
pmap_sync_page_data_phys(__unused ppnum_t pa)
{
return;
}
void
pmap_sync_page_attributes_phys(ppnum_t pa)
{
cache_flush_page_phys(pa);
}
int collect_ref;
int collect_unref;
void
pmap_collect(
pmap_t p)
{
register pt_entry_t *pdp, *ptp;
pt_entry_t *eptp;
int wired;
spl_t spl;
if (p == PMAP_NULL)
return;
if (p == kernel_pmap)
return;
PMAP_READ_LOCK(p, spl);
for (pdp = (pt_entry_t *)p->dirbase;
pdp < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)];
pdp++)
{
if (*pdp & INTEL_PTE_VALID) {
if(*pdp & INTEL_PTE_REF) {
*pdp &= ~INTEL_PTE_REF;
collect_ref++;
} else {
collect_unref++;
ptp = pmap_pte(p, pdetova(pdp - (pt_entry_t *)p->dirbase));
eptp = ptp + NPTEPG;
wired = 0;
{
register pt_entry_t *ptep;
for (ptep = ptp; ptep < eptp; ptep++) {
if (iswired(*ptep)) {
wired = 1;
break;
}
}
}
if (!wired) {
pmap_remove_range(p,
pdetova(pdp - (pt_entry_t *)p->dirbase),
ptp,
eptp);
*pdp = 0x0;
PMAP_READ_UNLOCK(p, spl);
{
register vm_page_t m;
vm_object_lock(p->pm_obj);
m = vm_page_lookup(p->pm_obj,(vm_object_offset_t)(pdp - (pt_entry_t *)&p->dirbase[0]));
if (m == VM_PAGE_NULL)
panic("pmap_collect: pte page not in object");
vm_page_lock_queues();
vm_page_free(m);
inuse_ptepages_count--;
vm_page_unlock_queues();
vm_object_unlock(p->pm_obj);
}
PMAP_READ_LOCK(p, spl);
}
}
}
}
PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
PMAP_READ_UNLOCK(p, spl);
return;
}
#if 0
pmap_t
pmap_kernel(void)
{
return (kernel_pmap);
}
#endif/* 0 */
void
pmap_copy_page(src, dst)
ppnum_t src;
ppnum_t dst;
{
bcopy_phys((addr64_t)i386_ptob(src),
(addr64_t)i386_ptob(dst),
PAGE_SIZE);
}
void
pmap_pageable(
__unused pmap_t pmap,
__unused vm_offset_t start_addr,
__unused vm_offset_t end_addr,
__unused boolean_t pageable)
{
#ifdef lint
pmap++; start_addr++; end_addr++; pageable++;
#endif
}
void
phys_attribute_clear(
ppnum_t pn,
int bits)
{
pv_entry_t pv_h;
register pv_entry_t pv_e;
register pt_entry_t *pte;
int pai;
register pmap_t pmap;
spl_t spl;
pmap_paddr_t phys;
assert(pn != vm_page_fictitious_addr);
if (!valid_page(pn)) {
return;
}
PMAP_WRITE_LOCK(spl);
phys = i386_ptob(pn);
pai = pa_index(phys);
pv_h = pai_to_pvh(pai);
if (pv_h->pmap != PMAP_NULL) {
for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
pmap = pv_e->pmap;
simple_lock(&pmap->lock);
{
register vm_offset_t va;
va = pv_e->va;
pte = pmap_pte(pmap, va);
#if 0
assert(*pte & INTEL_PTE_VALID);
#endif
*pte++ &= ~bits;
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
}
simple_unlock(&pmap->lock);
}
}
pmap_phys_attributes[pai] &= ~bits;
PMAP_WRITE_UNLOCK(spl);
}
boolean_t
phys_attribute_test(
ppnum_t pn,
int bits)
{
pv_entry_t pv_h;
register pv_entry_t pv_e;
register pt_entry_t *pte;
int pai;
register pmap_t pmap;
spl_t spl;
pmap_paddr_t phys;
assert(pn != vm_page_fictitious_addr);
if (!valid_page(pn)) {
return (FALSE);
}
PMAP_WRITE_LOCK(spl);
phys = i386_ptob(pn);
pai = pa_index(phys);
pv_h = pai_to_pvh(pai);
if (pmap_phys_attributes[pai] & bits) {
PMAP_WRITE_UNLOCK(spl);
return (TRUE);
}
if (pv_h->pmap != PMAP_NULL) {
for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
pmap = pv_e->pmap;
simple_lock(&pmap->lock);
{
register vm_offset_t va;
va = pv_e->va;
pte = pmap_pte(pmap, va);
#if 0
assert(*pte & INTEL_PTE_VALID);
#endif
}
{
if (*pte++ & bits) {
simple_unlock(&pmap->lock);
PMAP_WRITE_UNLOCK(spl);
return (TRUE);
}
}
simple_unlock(&pmap->lock);
}
}
PMAP_WRITE_UNLOCK(spl);
return (FALSE);
}
void
phys_attribute_set(
ppnum_t pn,
int bits)
{
int spl;
pmap_paddr_t phys;
assert(pn != vm_page_fictitious_addr);
if (!valid_page(pn)) {
return;
}
phys = i386_ptob(pn);
PMAP_WRITE_LOCK(spl);
pmap_phys_attributes[pa_index(phys)] |= bits;
PMAP_WRITE_UNLOCK(spl);
}
void pmap_set_modify(
ppnum_t pn)
{
phys_attribute_set(pn, PHYS_MODIFIED);
}
void
pmap_clear_modify(
ppnum_t pn)
{
phys_attribute_clear(pn, PHYS_MODIFIED);
}
boolean_t
pmap_is_modified(
ppnum_t pn)
{
return (phys_attribute_test(pn, PHYS_MODIFIED));
}
void
pmap_clear_reference(
ppnum_t pn)
{
phys_attribute_clear(pn, PHYS_REFERENCED);
}
void
pmap_set_reference(ppnum_t pn)
{
phys_attribute_set(pn, PHYS_REFERENCED);
}
boolean_t
pmap_is_referenced(
ppnum_t pn)
{
return (phys_attribute_test(pn, PHYS_REFERENCED));
}
unsigned int
pmap_get_refmod(ppnum_t pa)
{
return ( ((phys_attribute_test(pa, PHYS_MODIFIED))? VM_MEM_MODIFIED : 0)
| ((phys_attribute_test(pa, PHYS_REFERENCED))? VM_MEM_REFERENCED : 0));
}
void
pmap_clear_refmod(ppnum_t pa, unsigned int mask)
{
unsigned int x86Mask;
x86Mask = ( ((mask & VM_MEM_MODIFIED)? PHYS_MODIFIED : 0)
| ((mask & VM_MEM_REFERENCED)? PHYS_REFERENCED : 0));
phys_attribute_clear(pa, x86Mask);
}
void
pmap_modify_pages(
pmap_t map,
vm_offset_t s,
vm_offset_t e)
{
spl_t spl;
register pt_entry_t *pde;
register pt_entry_t *spte, *epte;
vm_offset_t l;
vm_offset_t orig_s = s;
if (map == PMAP_NULL)
return;
PMAP_READ_LOCK(map, spl);
pde = pmap_pde(map, s);
while (s && s < e) {
l = (s + PDE_MAPPED_SIZE) & ~(PDE_MAPPED_SIZE-1);
if (l > e)
l = e;
if (*pde & INTEL_PTE_VALID) {
spte = (pt_entry_t *)pmap_pte(map, (s & ~(PDE_MAPPED_SIZE-1)));
if (l) {
spte = &spte[ptenum(s)];
epte = &spte[intel_btop(l-s)];
} else {
epte = &spte[intel_btop(PDE_MAPPED_SIZE)];
spte = &spte[ptenum(s)];
}
while (spte < epte) {
if (*spte & INTEL_PTE_VALID) {
*spte |= (INTEL_PTE_MOD | INTEL_PTE_WRITE);
}
spte++;
}
}
s = l;
pde++;
}
PMAP_UPDATE_TLBS(map, orig_s, e);
PMAP_READ_UNLOCK(map, spl);
}
void
invalidate_icache(__unused vm_offset_t addr,
__unused unsigned cnt,
__unused int phys)
{
return;
}
void
flush_dcache(__unused vm_offset_t addr,
__unused unsigned count,
__unused int phys)
{
return;
}
void
signal_cpus(
cpu_set use_list,
pmap_t pmap,
vm_offset_t start_addr,
vm_offset_t end_addr)
{
register int which_cpu, j;
register pmap_update_list_t update_list_p;
while ((which_cpu = ffs((unsigned long)use_list)) != 0) {
which_cpu -= 1;
update_list_p = cpu_update_list(which_cpu);
simple_lock(&update_list_p->lock);
j = update_list_p->count;
if (j >= UPDATE_LIST_SIZE) {
update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS;
update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS;
}
else {
update_list_p->item[j].pmap = pmap;
update_list_p->item[j].start = start_addr;
update_list_p->item[j].end = end_addr;
update_list_p->count = j+1;
}
cpu_update_needed(which_cpu) = TRUE;
simple_unlock(&update_list_p->lock);
if (((cpus_idle & (1 << which_cpu)) == 0) ||
(pmap == kernel_pmap) || PMAP_REAL(which_cpu) == pmap)
{
i386_signal_cpu(which_cpu, MP_TLB_FLUSH, ASYNC);
}
use_list &= ~(1 << which_cpu);
}
}
void
process_pmap_updates(
register pmap_t my_pmap)
{
register int my_cpu;
register pmap_update_list_t update_list_p;
register int j;
register pmap_t pmap;
mp_disable_preemption();
my_cpu = cpu_number();
update_list_p = cpu_update_list(my_cpu);
simple_lock(&update_list_p->lock);
for (j = 0; j < update_list_p->count; j++) {
pmap = update_list_p->item[j].pmap;
if (pmap == my_pmap ||
pmap == kernel_pmap) {
if (pmap->ref_count <= 0) {
PMAP_CPU_CLR(pmap, my_cpu);
PMAP_REAL(my_cpu) = kernel_pmap;
#ifdef PAE
set_cr3((unsigned int)kernel_pmap->pm_ppdpt);
#else
set_cr3((unsigned int)kernel_pmap->pdirbase);
#endif
} else
INVALIDATE_TLB(pmap,
update_list_p->item[j].start,
update_list_p->item[j].end);
}
}
update_list_p->count = 0;
cpu_update_needed(my_cpu) = FALSE;
simple_unlock(&update_list_p->lock);
mp_enable_preemption();
}
void
pmap_update_interrupt(void)
{
register int my_cpu;
spl_t s;
register pmap_t my_pmap;
mp_disable_preemption();
my_cpu = cpu_number();
s = splhigh();
my_pmap = PMAP_REAL(my_cpu);
if (!(my_pmap && pmap_in_use(my_pmap, my_cpu)))
my_pmap = kernel_pmap;
do {
LOOP_VAR;
i_bit_clear(my_cpu, &cpus_active);
while (*(volatile int *)(&my_pmap->lock.interlock.lock_data) ||
*(volatile int *)(&kernel_pmap->lock.interlock.lock_data)) {
LOOP_CHECK("pmap_update_interrupt", my_pmap);
cpu_pause();
}
process_pmap_updates(my_pmap);
i_bit_set(my_cpu, &cpus_active);
} while (cpu_update_needed(my_cpu));
splx(s);
mp_enable_preemption();
}
#if MACH_KDB
extern void db_show_page(pmap_paddr_t pa);
void
db_show_page(pmap_paddr_t pa)
{
pv_entry_t pv_h;
int pai;
char attr;
pai = pa_index(pa);
pv_h = pai_to_pvh(pai);
attr = pmap_phys_attributes[pai];
printf("phys page %x ", pa);
if (attr & PHYS_MODIFIED)
printf("modified, ");
if (attr & PHYS_REFERENCED)
printf("referenced, ");
if (pv_h->pmap || pv_h->next)
printf(" mapped at\n");
else
printf(" not mapped\n");
for (; pv_h; pv_h = pv_h->next)
if (pv_h->pmap)
printf("%x in pmap %x\n", pv_h->va, pv_h->pmap);
}
#endif
#if MACH_KDB
void db_kvtophys(vm_offset_t);
void db_show_vaddrs(pt_entry_t *);
void
db_kvtophys(
vm_offset_t vaddr)
{
db_printf("0x%x", kvtophys(vaddr));
}
void
db_show_vaddrs(
pt_entry_t *dirbase)
{
pt_entry_t *ptep, *pdep, tmp;
int x, y, pdecnt, ptecnt;
if (dirbase == 0) {
dirbase = kernel_pmap->dirbase;
}
if (dirbase == 0) {
db_printf("need a dirbase...\n");
return;
}
dirbase = (pt_entry_t *) ((unsigned long) dirbase & ~INTEL_OFFMASK);
db_printf("dirbase: 0x%x\n", dirbase);
pdecnt = ptecnt = 0;
pdep = &dirbase[0];
for (y = 0; y < NPDEPG; y++, pdep++) {
if (((tmp = *pdep) & INTEL_PTE_VALID) == 0) {
continue;
}
pdecnt++;
ptep = (pt_entry_t *) ((*pdep) & ~INTEL_OFFMASK);
db_printf("dir[%4d]: 0x%x\n", y, *pdep);
for (x = 0; x < NPTEPG; x++, ptep++) {
if (((tmp = *ptep) & INTEL_PTE_VALID) == 0) {
continue;
}
ptecnt++;
db_printf(" tab[%4d]: 0x%x, va=0x%x, pa=0x%x\n",
x,
*ptep,
(y << 22) | (x << 12),
*ptep & ~INTEL_OFFMASK);
}
}
db_printf("total: %d tables, %d page table entries.\n", pdecnt, ptecnt);
}
#endif
#include <mach_vm_debug.h>
#if MACH_VM_DEBUG
#include <vm/vm_debug.h>
int
pmap_list_resident_pages(
__unused pmap_t pmap,
__unused vm_offset_t *listp,
__unused int space)
{
return 0;
}
#endif
#ifdef MACH_BSD
void
pmap_movepage(unsigned long from, unsigned long to, vm_size_t size)
{
spl_t spl;
pt_entry_t *pte, saved_pte;
PMAP_READ_LOCK(kernel_pmap, spl);
while (size > 0) {
pte = pmap_pte(kernel_pmap, from);
if (pte == NULL)
panic("pmap_pagemove from pte NULL");
saved_pte = *pte;
PMAP_READ_UNLOCK(kernel_pmap, spl);
pmap_enter(kernel_pmap, to, (ppnum_t)i386_btop(i386_trunc_page(*pte)),
VM_PROT_READ|VM_PROT_WRITE, 0, *pte & INTEL_PTE_WIRED);
pmap_remove(kernel_pmap, (addr64_t)from, (addr64_t)(from+PAGE_SIZE));
PMAP_READ_LOCK(kernel_pmap, spl);
pte = pmap_pte(kernel_pmap, to);
if (pte == NULL)
panic("pmap_pagemove 'to' pte NULL");
*pte = saved_pte;
from += PAGE_SIZE;
to += PAGE_SIZE;
size -= PAGE_SIZE;
}
PMAP_UPDATE_TLBS(kernel_pmap, from, from+size);
PMAP_UPDATE_TLBS(kernel_pmap, to, to+size);
PMAP_READ_UNLOCK(kernel_pmap, spl);
}
#endif
boolean_t
coredumpok(vm_map_t map, vm_offset_t va)
{
pt_entry_t *ptep;
ptep = pmap_pte(map->pmap, va);
if (0 == ptep)
return FALSE;
return ((*ptep & (INTEL_PTE_NCACHE | INTEL_PTE_WIRED)) != (INTEL_PTE_NCACHE | INTEL_PTE_WIRED));
}
void
pmap_growkernel(vm_offset_t addr)
{
#if GROW_KERNEL_FUNCTION_IMPLEMENTED
struct pmap *pmap;
int s;
vm_offset_t ptppaddr;
ppnum_t ppn;
vm_page_t nkpg;
pd_entry_t newpdir = 0;
if (kptobj == 0) panic("growkernel 0");
if (!vm_object_lock_try(kptobj)) {
return;
}
vm_page_lock_queues();
s = splhigh();
if (kernel_vm_end == 0) {
kernel_vm_end = KERNBASE;
nkpt = 0;
while (pdir_pde(kernel_pmap->dirbase, kernel_vm_end)) {
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
nkpt++;
}
}
addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
while (kernel_vm_end < addr) {
if (pdir_pde(kernel_pmap->dirbase, kernel_vm_end)) {
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
continue;
}
nkpg = vm_page_alloc(kptobj, nkpt);
if (!nkpg)
panic("pmap_growkernel: no memory to grow kernel");
nkpt++;
vm_page_wire(nkpg);
ppn = nkpg->phys_page;
pmap_zero_page(ppn);
ptppaddr = i386_ptob(ppn);
newpdir = (pd_entry_t) (ptppaddr | INTEL_PTE_VALID |
INTEL_PTE_RW | INTEL_PTE_REF | INTEL_PTE_MOD);
pdir_pde(kernel_pmap->dirbase, kernel_vm_end) = newpdir;
simple_lock(&free_pmap_lock);
for (pmap = (struct pmap *)kernel_pmap->pmap_link.next;
pmap != kernel_pmap ;
pmap = (struct pmap *)pmap->pmap_link.next ) {
*pmap_pde(pmap, kernel_vm_end) = newpdir;
}
simple_unlock(&free_pmap_lock);
}
splx(s);
vm_page_unlock_queues();
vm_object_unlock(kptobj);
#endif
}
pt_entry_t *
pmap_mapgetpte(vm_map_t map, vm_offset_t v)
{
return pmap_pte(map->pmap, v);
}
boolean_t
phys_page_exists(
ppnum_t pn)
{
pmap_paddr_t phys;
assert(pn != vm_page_fictitious_addr);
if (!pmap_initialized)
return (TRUE);
phys = (pmap_paddr_t) i386_ptob(pn);
if (!pmap_valid_page(pn))
return (FALSE);
return TRUE;
}
void
mapping_free_prime()
{
int i;
pv_entry_t pv_e;
for (i = 0; i < (5 * PV_ALLOC_CHUNK); i++) {
pv_e = (pv_entry_t) zalloc(pv_list_zone);
PV_FREE(pv_e);
}
}
void
mapping_adjust()
{
pv_entry_t pv_e;
int i;
int spl;
if (mapping_adjust_call == NULL) {
thread_call_setup(&mapping_adjust_call_data,
(thread_call_func_t) mapping_adjust,
(thread_call_param_t) NULL);
mapping_adjust_call = &mapping_adjust_call_data;
}
if (pv_free_count < PV_LOW_WATER_MARK) {
for (i = 0; i < PV_ALLOC_CHUNK; i++) {
pv_e = (pv_entry_t) zalloc(pv_list_zone);
SPLVM(spl);
PV_FREE(pv_e);
SPLX(spl);
}
}
mappingrecurse = 0;
}
void
pmap_commpage_init(vm_offset_t kernel_commpage, vm_offset_t user_commpage, int cnt)
{
int i;
pt_entry_t *opte, *npte;
pt_entry_t pte;
for (i = 0; i < cnt; i++) {
opte = pmap_pte(kernel_pmap, kernel_commpage);
if (0 == opte) panic("kernel_commpage");
npte = pmap_pte(kernel_pmap, user_commpage);
if (0 == npte) panic("user_commpage");
pte = *opte | INTEL_PTE_USER|INTEL_PTE_GLOBAL;
pte &= ~INTEL_PTE_WRITE; WRITE_PTE_FAST(npte, pte);
kernel_commpage += INTEL_PGBYTES;
user_commpage += INTEL_PGBYTES;
}
}
static cpu_pmap_t cpu_pmap_master;
static struct pmap_update_list cpu_update_list_master;
struct cpu_pmap *
pmap_cpu_alloc(boolean_t is_boot_cpu)
{
int ret;
int i;
cpu_pmap_t *cp;
pmap_update_list_t up;
vm_offset_t address;
vm_map_entry_t entry;
if (is_boot_cpu) {
cp = &cpu_pmap_master;
up = &cpu_update_list_master;
} else {
ret = kmem_alloc(kernel_map,
(vm_offset_t *) &cp, sizeof(cpu_pmap_t));
if (ret != KERN_SUCCESS) {
printf("pmap_cpu_alloc() failed ret=%d\n", ret);
return NULL;
}
bzero((void *)cp, sizeof(cpu_pmap_t));
ret = kmem_alloc(kernel_map,
(vm_offset_t *) &up, sizeof(*up));
if (ret != KERN_SUCCESS) {
printf("pmap_cpu_alloc() failed ret=%d\n", ret);
pmap_cpu_free(cp);
return NULL;
}
for (i = 0; i < PMAP_NWINDOWS; i++) {
ret = vm_map_find_space(kernel_map,
&address, PAGE_SIZE, 0, &entry);
if (ret != KERN_SUCCESS) {
printf("pmap_cpu_alloc() "
"vm_map_find_space ret=%d\n", ret);
pmap_cpu_free(cp);
return NULL;
}
vm_map_unlock(kernel_map);
cp->mapwindow[i].prv_CADDR = (caddr_t) address;
cp->mapwindow[i].prv_CMAP = vtopte(address);
* (int *) cp->mapwindow[i].prv_CMAP = 0;
kprintf("pmap_cpu_alloc() "
"window=%d CADDR=0x%x CMAP=0x%x\n",
i, address, vtopte(address));
}
}
cp->update_list = up;
simple_lock_init(&up->lock, 0);
up->count = 0;
return cp;
}
void
pmap_cpu_free(struct cpu_pmap *cp)
{
if (cp != NULL && cp != &cpu_pmap_master) {
if (cp->update_list != NULL)
kfree((void *) cp->update_list,
sizeof(*cp->update_list));
kfree((void *) cp, sizeof(cpu_pmap_t));
}
}