#include <string.h>
#include <mach_assert.h>
#include <mach_ldebug.h>
#include <mach/shared_region.h>
#include <mach/vm_param.h>
#include <mach/vm_prot.h>
#include <mach/vm_map.h>
#include <mach/machine/vm_param.h>
#include <mach/machine/vm_types.h>
#include <mach/boolean.h>
#include <kern/thread.h>
#include <kern/sched.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
#include <kern/ledger.h>
#include <kern/misc_protos.h>
#include <kern/spl.h>
#include <kern/xpr.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_protos.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/cpm.h>
#include <libkern/section_keywords.h>
#include <machine/atomic.h>
#include <machine/thread.h>
#include <machine/lowglobals.h>
#include <arm/caches_internal.h>
#include <arm/cpu_data.h>
#include <arm/cpu_data_internal.h>
#include <arm/cpu_capabilities.h>
#include <arm/cpu_number.h>
#include <arm/machine_cpu.h>
#include <arm/misc_protos.h>
#include <arm/trap.h>
#include <libkern/section_keywords.h>
#if (__ARM_VMSA__ > 7)
#include <arm64/proc_reg.h>
#include <pexpert/arm64/boot.h>
#if CONFIG_PGTRACE
#include <stdint.h>
#include <arm64/pgtrace.h>
#if CONFIG_PGTRACE_NONKEXT
#include <arm64/pgtrace_decoder.h>
#endif // CONFIG_PGTRACE_NONKEXT
#endif
#endif
#include <pexpert/device_tree.h>
#include <san/kasan.h>
#if MACH_ASSERT
int pmap_stats_assert = 1;
#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...) \
MACRO_BEGIN \
if (pmap_stats_assert && (pmap)->pmap_stats_assert) \
assertf(cond, fmt, ##__VA_ARGS__); \
MACRO_END
#else
#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...)
#endif
#if DEVELOPMENT || DEBUG
#define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended)
#else
#define PMAP_FOOTPRINT_SUSPENDED(pmap) (FALSE)
#endif
#if DEVELOPMENT || DEBUG
int panic_on_unsigned_execute = 0;
#endif
#if (__ARM_VMSA__ == 7)
#define VREGION1_START (VM_HIGH_KERNEL_WINDOW & ~ARM_TT_L1_PT_OFFMASK)
#else
#define VREGION1_HIGH_WINDOW (PE_EARLY_BOOT_VA)
#define VREGION1_START ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VREGION1_HIGH_WINDOW)
#endif
#define VREGION1_SIZE (trunc_page(VM_MAX_KERNEL_ADDRESS - (VREGION1_START)))
extern unsigned int not_in_kdp;
extern vm_offset_t first_avail;
extern pmap_paddr_t avail_start;
extern pmap_paddr_t avail_end;
extern vm_offset_t virtual_space_start;
extern vm_offset_t virtual_space_end;
extern int hard_maxproc;
#if (__ARM_VMSA__ > 7)
#define PGTABLE_ADDR_BITS (64ULL - T0SZ_BOOT)
#define ARM64_TTBR0_MIN_ADDR (0ULL)
#define ARM64_TTBR0_MAX_ADDR (0ULL + (1ULL << PGTABLE_ADDR_BITS) - 1)
#define ARM64_TTBR1_MIN_ADDR (0ULL - (1ULL << PGTABLE_ADDR_BITS))
#define ARM64_TTBR1_MAX_ADDR (~0ULL)
const uint64_t arm64_root_pgtable_level = (3 - ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) / (ARM_PGSHIFT - TTE_SHIFT)));
const uint64_t arm64_root_pgtable_num_ttes = (2 << ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) % (ARM_PGSHIFT - TTE_SHIFT)));
#else
const uint64_t arm64_root_pgtable_level = 0;
const uint64_t arm64_root_pgtable_num_ttes = 0;
#endif
struct pmap kernel_pmap_store MARK_AS_PMAP_DATA;
SECURITY_READ_ONLY_LATE(pmap_t) kernel_pmap = &kernel_pmap_store;
struct vm_object pmap_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
vm_object_t pmap_object = &pmap_object_store;
static struct zone *pmap_zone;
decl_simple_lock_data(, pmaps_lock MARK_AS_PMAP_DATA)
unsigned int pmap_stamp MARK_AS_PMAP_DATA;
queue_head_t map_pmap_list MARK_AS_PMAP_DATA;
queue_head_t tt_pmap_list MARK_AS_PMAP_DATA;
unsigned int tt_pmap_count MARK_AS_PMAP_DATA;
unsigned int tt_pmap_max MARK_AS_PMAP_DATA;
decl_simple_lock_data(, pt_pages_lock MARK_AS_PMAP_DATA)
queue_head_t pt_page_list MARK_AS_PMAP_DATA;
decl_simple_lock_data(, pmap_pages_lock MARK_AS_PMAP_DATA)
typedef struct page_free_entry {
struct page_free_entry *next;
} page_free_entry_t;
#define PAGE_FREE_ENTRY_NULL ((page_free_entry_t *) 0)
page_free_entry_t *pmap_pages_reclaim_list MARK_AS_PMAP_DATA;
unsigned int pmap_pages_request_count MARK_AS_PMAP_DATA;
unsigned long long pmap_pages_request_acum MARK_AS_PMAP_DATA;
typedef struct tt_free_entry {
struct tt_free_entry *next;
} tt_free_entry_t;
#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0)
tt_free_entry_t *free_page_size_tt_list MARK_AS_PMAP_DATA;
unsigned int free_page_size_tt_count MARK_AS_PMAP_DATA;
unsigned int free_page_size_tt_max MARK_AS_PMAP_DATA;
#define FREE_PAGE_SIZE_TT_MAX 4
tt_free_entry_t *free_two_page_size_tt_list MARK_AS_PMAP_DATA;
unsigned int free_two_page_size_tt_count MARK_AS_PMAP_DATA;
unsigned int free_two_page_size_tt_max MARK_AS_PMAP_DATA;
#define FREE_TWO_PAGE_SIZE_TT_MAX 4
tt_free_entry_t *free_tt_list MARK_AS_PMAP_DATA;
unsigned int free_tt_count MARK_AS_PMAP_DATA;
unsigned int free_tt_max MARK_AS_PMAP_DATA;
#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0)
boolean_t pmap_gc_allowed MARK_AS_PMAP_DATA = TRUE;
boolean_t pmap_gc_forced MARK_AS_PMAP_DATA = FALSE;
boolean_t pmap_gc_allowed_by_time_throttle = TRUE;
unsigned int inuse_user_ttepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_user_ptepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_user_tteroot_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_kernel_ttepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_kernel_ptepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_kernel_tteroot_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_pmap_pages_count = 0;
SECURITY_READ_ONLY_LATE(tt_entry_t *) invalid_tte = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) invalid_ttep = 0;
SECURITY_READ_ONLY_LATE(tt_entry_t *) cpu_tte = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) cpu_ttep = 0;
#if DEVELOPMENT || DEBUG
int nx_enabled = 1;
int allow_data_exec = 0;
int allow_stack_exec = 0;
#else
const int nx_enabled = 1;
const int allow_data_exec = 0;
const int allow_stack_exec = 0;
#endif
typedef struct pv_entry {
struct pv_entry *pve_next;
pt_entry_t *pve_ptep;
#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
} __attribute__ ((aligned(8))) pv_entry_t;
#else
} pv_entry_t;
#endif
#define PV_ENTRY_NULL ((pv_entry_t *) 0)
#define PVE_NEXT_ALTACCT ((uintptr_t) 0x1)
#define PVE_NEXT_SET_ALTACCT(pve_next_p) \
*(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) | \
PVE_NEXT_ALTACCT)
#define PVE_NEXT_CLR_ALTACCT(pve_next_p) \
*(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) & \
~PVE_NEXT_ALTACCT)
#define PVE_NEXT_IS_ALTACCT(pve_next) \
((((uintptr_t) (pve_next)) & PVE_NEXT_ALTACCT) ? TRUE : FALSE)
#define PVE_NEXT_PTR(pve_next) \
((struct pv_entry *)(((uintptr_t) (pve_next)) & \
~PVE_NEXT_ALTACCT))
#if MACH_ASSERT
static void pmap_check_ledgers(pmap_t pmap);
#else
static inline void pmap_check_ledgers(__unused pmap_t pmap) {}
#endif
SECURITY_READ_ONLY_LATE(pv_entry_t **) pv_head_table;
pv_entry_t *pv_free_list MARK_AS_PMAP_DATA;
pv_entry_t *pv_kern_free_list MARK_AS_PMAP_DATA;
decl_simple_lock_data(,pv_free_list_lock MARK_AS_PMAP_DATA)
decl_simple_lock_data(,pv_kern_free_list_lock MARK_AS_PMAP_DATA)
decl_simple_lock_data(,phys_backup_lock)
#if (__ARM_VMSA__ == 7)
#define PT_INDEX_MAX 1
#else
#if (ARM_PGSHIFT == 14)
#define PT_INDEX_MAX 1
#else
#define PT_INDEX_MAX 4
#endif
#endif
#define PT_DESC_REFCOUNT 0x4000U
typedef struct pt_desc {
queue_chain_t pt_page;
struct {
unsigned short refcnt;
unsigned short wiredcnt;
} pt_cnt[PT_INDEX_MAX];
struct pmap *pmap;
struct {
vm_offset_t va;
} pt_map[PT_INDEX_MAX];
} pt_desc_t;
#define PTD_ENTRY_NULL ((pt_desc_t *) 0)
SECURITY_READ_ONLY_LATE(pt_desc_t *) ptd_root_table;
pt_desc_t *ptd_free_list MARK_AS_PMAP_DATA = PTD_ENTRY_NULL;
SECURITY_READ_ONLY_LATE(boolean_t) ptd_preboot = TRUE;
unsigned int ptd_free_count MARK_AS_PMAP_DATA = 0;
decl_simple_lock_data(,ptd_free_list_lock MARK_AS_PMAP_DATA)
typedef u_int16_t pp_attr_t;
#define PP_ATTR_WIMG_MASK 0x003F
#define PP_ATTR_WIMG(x) ((x) & PP_ATTR_WIMG_MASK)
#define PP_ATTR_REFERENCED 0x0040
#define PP_ATTR_MODIFIED 0x0080
#define PP_ATTR_INTERNAL 0x0100
#define PP_ATTR_REUSABLE 0x0200
#define PP_ATTR_ALTACCT 0x0400
#define PP_ATTR_NOENCRYPT 0x0800
#define PP_ATTR_REFFAULT 0x1000
#define PP_ATTR_MODFAULT 0x2000
SECURITY_READ_ONLY_LATE(pp_attr_t*) pp_attr_table;
typedef uint8_t io_attr_t;
#define IO_ATTR_WIMG_MASK 0x3F
#define IO_ATTR_WIMG(x) ((x) & IO_ATTR_WIMG_MASK)
SECURITY_READ_ONLY_LATE(io_attr_t*) io_attr_table;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_first_phys = (pmap_paddr_t) 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_last_phys = (pmap_paddr_t) 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_start = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_end = 0;
SECURITY_READ_ONLY_LATE(uint32_t) io_rgn_granule = 0;
SECURITY_READ_ONLY_LATE(boolean_t) pmap_initialized = FALSE;
SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_min;
SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_max;
SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm_pmap_max_offset_default = 0x0;
#if defined(__arm64__)
SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0;
#endif
static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA;
#if (__ARM_VMSA__ > 7)
SECURITY_READ_ONLY_LATE(pmap_t) sharedpage_pmap;
#endif
#define pa_index(pa) \
(atop((pa) - vm_first_phys))
#define pai_to_pvh(pai) \
(&pv_head_table[pai])
#define pa_valid(x) \
((x) >= vm_first_phys && (x) < vm_last_phys)
#define pte_is_wired(pte) \
(((pte) & ARM_PTE_WIRED_MASK) == ARM_PTE_WIRED)
#define pte_set_wired(ptep, wired) \
do { \
SInt16 *ptd_wiredcnt_ptr; \
ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(ptep)->pt_cnt[ARM_PT_DESC_INDEX(ptep)].wiredcnt); \
if (wired) { \
*ptep |= ARM_PTE_WIRED; \
OSAddAtomic16(1, ptd_wiredcnt_ptr); \
} else { \
*ptep &= ~ARM_PTE_WIRED; \
OSAddAtomic16(-1, ptd_wiredcnt_ptr); \
} \
} while(0)
#define pte_is_ffr(pte) \
(((pte) & ARM_PTE_WRITEABLE) == ARM_PTE_WRITEABLE)
#define pte_set_ffr(pte, ffr) \
do { \
if (ffr) { \
pte |= ARM_PTE_WRITEABLE; \
} else { \
pte &= ~ARM_PTE_WRITEABLE; \
} \
} while(0)
#define pve_next(pve) \
((pve)->pve_next)
#define pve_link_field(pve) \
(&pve_next(pve))
#define pve_link(pp, e) \
((pve_next(e) = pve_next(pp)), (pve_next(pp) = (e)))
#define pve_unlink(pp, e) \
(pve_next(pp) = pve_next(e))
#define pve_get_ptep(pve) \
((pve)->pve_ptep)
#define pve_set_ptep(pve, ptep_new) \
do { \
(pve)->pve_ptep = (ptep_new); \
} while (0)
#if (__ARM_VMSA__ == 7)
#define ARM_PT_DESC_INDEX_MASK 0x00000
#define ARM_PT_DESC_INDEX_SHIFT 0
#define ARM_TT_PT_INDEX_MASK 0xfffU
#define ARM_TT_PT_ADDR_SHIFT (10U)
#define ARM_PT_DESC_INDEX(ptep) \
(((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT)
#define ptep_get_ptd(ptep) \
((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK))
#define ptep_get_va(ptep) \
((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<<ARM_TT_PT_ADDR_SHIFT))
#define ptep_get_pmap(ptep) \
((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pmap))
#else
#if (ARM_PGSHIFT == 12)
#define ARM_PT_DESC_INDEX_MASK ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0x00000ULL : 0x03000ULL)
#define ARM_PT_DESC_INDEX_SHIFT ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0 : 12)
#define ARM_TT_PT_INDEX_MASK (0x0fffULL)
#define ARM_TT_PT_ADDR_SHIFT (9ULL)
#define ARM_TT_PT_OTHER_MASK (0x0fffULL)
#else
#define ARM_PT_DESC_INDEX_MASK (0x00000)
#define ARM_PT_DESC_INDEX_SHIFT (0)
#define ARM_TT_PT_INDEX_MASK (0x3fffULL)
#define ARM_TT_PT_ADDR_SHIFT (11ULL)
#define ARM_TT_PT_OTHER_MASK (0x3fffULL)
#endif
#define ARM_PT_DESC_INDEX(ptep) \
(((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT)
#define ptep_get_ptd(ptep) \
((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK))
#define ptep_get_va(ptep) \
((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<<ARM_TT_PT_ADDR_SHIFT))
#define ptep_get_pmap(ptep) \
((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pmap))
#endif
#define PVH_TYPE_NULL 0x0UL
#define PVH_TYPE_PVEP 0x1UL
#define PVH_TYPE_PTEP 0x2UL
#define PVH_TYPE_PTDP 0x3UL
#define PVH_TYPE_MASK (0x3UL)
#define PVH_LIST_MASK (~PVH_TYPE_MASK)
#if (__ARM_VMSA__ == 7)
#define pvh_set_bits(h, b) \
do { \
while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | (b), (vm_offset_t *)(h))); \
} while (0)
#define pvh_clear_bits(h, b) \
do { \
while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~(b), (vm_offset_t *)(h))); \
} while (0)
#else
#define pvh_set_bits(h, b) \
do { \
while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | ((int64_t)b), (vm_offset_t *)(h))); \
} while (0)
#define pvh_clear_bits(h, b) \
do { \
while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~((int64_t)b), (vm_offset_t *)(h))); \
} while (0)
#endif
#define pvh_test_type(h, b) \
((*(vm_offset_t *)(h) & (PVH_TYPE_MASK)) == (b))
#define pvh_ptep(h) \
((pt_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK))
#define pvh_list(h) \
((pv_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK))
#define pvh_bits(h) \
(*(vm_offset_t *)(h) & PVH_TYPE_MASK)
#if (__ARM_VMSA__ == 7)
#define pvh_update_head(h, e, t) \
do { \
while (!OSCompareAndSwap(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \
} while (0)
#else
#define pvh_update_head(h, e, t) \
do { \
while (!OSCompareAndSwap64(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \
} while (0)
#endif
#define pvh_add(h, e) \
do { \
assert(!pvh_test_type((h), PVH_TYPE_PTEP)); \
pve_next(e) = pvh_list(h); \
pvh_update_head((h), (e), PVH_TYPE_PVEP); \
} while (0)
#define pvh_remove(h, p, e) \
do { \
assert(!PVE_NEXT_IS_ALTACCT(pve_next((e)))); \
if ((p) == (h)) { \
if (PVE_NEXT_PTR(pve_next((e))) == PV_ENTRY_NULL) { \
pvh_update_head((h), PV_ENTRY_NULL, PVH_TYPE_NULL); \
} else { \
pvh_update_head((h), PVE_NEXT_PTR(pve_next((e))), PVH_TYPE_PVEP); \
} \
} else { \
\
boolean_t __is_altacct; \
__is_altacct = PVE_NEXT_IS_ALTACCT(*(p)); \
*(p) = PVE_NEXT_PTR(pve_next((e))); \
if (__is_altacct) { \
PVE_NEXT_SET_ALTACCT((p)); \
} else { \
PVE_NEXT_CLR_ALTACCT((p)); \
} \
} \
} while (0)
#define ppattr_set_bits(h, b) \
do { \
while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) | (b), (pp_attr_t *)(h))); \
} while (0)
#define ppattr_clear_bits(h, b) \
do { \
while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) & ~(b), (pp_attr_t *)(h))); \
} while (0)
#define ppattr_test_bits(h, b) \
((*(pp_attr_t *)(h) & (b)) == (b))
#define pa_set_bits(x, b) \
do { \
if (pa_valid(x)) \
ppattr_set_bits(&pp_attr_table[pa_index(x)], \
(b)); \
} while (0)
#define pa_test_bits(x, b) \
(pa_valid(x) ? ppattr_test_bits(&pp_attr_table[pa_index(x)],\
(b)) : FALSE)
#define pa_clear_bits(x, b) \
do { \
if (pa_valid(x)) \
ppattr_clear_bits(&pp_attr_table[pa_index(x)], \
(b)); \
} while (0)
#define pa_set_modify(x) \
pa_set_bits(x, PP_ATTR_MODIFIED)
#define pa_clear_modify(x) \
pa_clear_bits(x, PP_ATTR_MODIFIED)
#define pa_set_reference(x) \
pa_set_bits(x, PP_ATTR_REFERENCED)
#define pa_clear_reference(x) \
pa_clear_bits(x, PP_ATTR_REFERENCED)
#define IS_INTERNAL_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
#define SET_INTERNAL_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
#define CLR_INTERNAL_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
#define IS_REUSABLE_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
#define SET_REUSABLE_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
#define CLR_REUSABLE_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
#define IS_ALTACCT_PAGE(pai, pve_p) \
(((pve_p) == NULL) \
? ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT) \
: PVE_NEXT_IS_ALTACCT(pve_next((pve_p))))
#define SET_ALTACCT_PAGE(pai, pve_p) \
if ((pve_p) == NULL) { \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT); \
} else { \
PVE_NEXT_SET_ALTACCT(&pve_next((pve_p))); \
}
#define CLR_ALTACCT_PAGE(pai, pve_p) \
if ((pve_p) == NULL) { \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT);\
} else { \
PVE_NEXT_CLR_ALTACCT(&pve_next((pve_p))); \
}
#define IS_REFFAULT_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
#define SET_REFFAULT_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
#define CLR_REFFAULT_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
#define IS_MODFAULT_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
#define SET_MODFAULT_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
#define CLR_MODFAULT_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
#if (__ARM_VMSA__ == 7)
#define tte_index(pmap, addr) \
ttenum((addr))
#define tte_get_ptd(tte) \
((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK))
#else
#define tt0_index(pmap, addr) \
(((addr) & ARM_TT_L0_INDEX_MASK) >> ARM_TT_L0_SHIFT)
#define tt1_index(pmap, addr) \
(((addr) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
#define tt2_index(pmap, addr) \
(((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
#define tt3_index(pmap, addr) \
(((addr) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
#define tte_index(pmap, addr) \
(((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
#define tte_get_ptd(tte) \
((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK))
#endif
#define PMAP_LOCK_INIT(pmap) { \
simple_lock_init(&(pmap)->lock, 0); \
}
#define PMAP_LOCK(pmap) { \
simple_lock(&(pmap)->lock); \
}
#define PMAP_UNLOCK(pmap) { \
simple_unlock(&(pmap)->lock); \
}
#if MACH_ASSERT
#define PMAP_ASSERT_LOCKED(pmap) { \
simple_lock_assert(&(pmap)->lock, LCK_ASSERT_OWNED); \
}
#else
#define PMAP_ASSERT_LOCKED(pmap)
#endif
#define LOCK_PVH(index) { \
hw_lock_bit((hw_lock_bit_t *) \
((unsigned int*)pv_head_table)-1-(index>>5), \
(index&0x1F)); \
}
#define UNLOCK_PVH(index) { \
hw_unlock_bit((hw_lock_bit_t *) \
((unsigned int*)pv_head_table)-1-(index>>5), \
(index&0x1F)); \
}
#define ASSERT_PVH_LOCKED(index) { \
assert(*(((unsigned int*)pv_head_table)-1-(index>>5)) & (1 << (index & 0x1F))); \
}
#define PMAP_UPDATE_TLBS(pmap, s, e) { \
flush_mmu_tlb_region_asid(s, (unsigned)(e - s), pmap); \
}
#ifdef __ARM_L1_PTW__
#define FLUSH_PTE_RANGE(spte, epte) \
__asm__ volatile("dsb ish");
#define FLUSH_PTE(pte_p) \
__asm__ volatile("dsb ish");
#else
#define FLUSH_PTE_RANGE(spte, epte) \
CleanPoU_DcacheRegion((vm_offset_t)spte, \
(vm_offset_t)epte - (vm_offset_t)spte);
#define FLUSH_PTE(pte_p) \
CleanPoU_DcacheRegion((vm_offset_t)pte_p, sizeof(pt_entry_t));
#endif
#define WRITE_PTE(pte_p, pte_entry) \
__unreachable_ok_push \
if (TEST_PAGE_RATIO_4) { \
do { \
if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \
if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \
*(pte_p) = (pte_entry); \
*((pte_p)+1) = (pte_entry); \
*((pte_p)+2) = (pte_entry); \
*((pte_p)+3) = (pte_entry); \
} else { \
*(pte_p) = (pte_entry); \
*((pte_p)+1) = (pte_entry) | 0x1000; \
*((pte_p)+2) = (pte_entry) | 0x2000; \
*((pte_p)+3) = (pte_entry) | 0x3000; \
} \
FLUSH_PTE_RANGE((pte_p),((pte_p)+4)); \
} while(0); \
} else { \
do { \
*(pte_p) = (pte_entry); \
FLUSH_PTE(pte_p); \
} while(0); \
} \
__unreachable_ok_pop
#define WRITE_PTE_FAST(pte_p, pte_entry) \
__unreachable_ok_push \
if (TEST_PAGE_RATIO_4) { \
if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \
if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \
*(pte_p) = (pte_entry); \
*((pte_p)+1) = (pte_entry); \
*((pte_p)+2) = (pte_entry); \
*((pte_p)+3) = (pte_entry); \
} else { \
*(pte_p) = (pte_entry); \
*((pte_p)+1) = (pte_entry) | 0x1000; \
*((pte_p)+2) = (pte_entry) | 0x2000; \
*((pte_p)+3) = (pte_entry) | 0x3000; \
} \
} else { \
*(pte_p) = (pte_entry); \
} \
__unreachable_ok_pop
#define current_pmap() \
(vm_map_pmap(current_thread()->map))
#define PMAP_IS_VALID(x) (TRUE)
#ifdef PMAP_TRACES
unsigned int pmap_trace = 0;
#define PMAP_TRACE(...) \
if (pmap_trace) { \
KDBG_RELEASE(__VA_ARGS__); \
}
#else
#define PMAP_TRACE(...) KDBG_DEBUG(__VA_ARGS__)
#endif
#define PMAP_TRACE_CONSTANT(...) KDBG_RELEASE(__VA_ARGS__)
static void pv_init(
void);
static boolean_t pv_alloc(
pmap_t pmap,
unsigned int pai,
pv_entry_t **pvepp);
static void pv_free(
pv_entry_t *pvep);
static void pv_list_free(
pv_entry_t *pvehp,
pv_entry_t *pvetp,
unsigned int cnt);
static void ptd_bootstrap(
pt_desc_t *ptdp, unsigned int ptd_cnt);
static pt_desc_t *ptd_alloc(
pmap_t pmap);
static void ptd_deallocate(
pt_desc_t *ptdp);
static void ptd_init(
pt_desc_t *ptdp, pmap_t pmap, vm_map_address_t va, unsigned int ttlevel, pt_entry_t * pte_p);
static void pmap_zone_init(
void);
static void pmap_set_reference(
ppnum_t pn);
ppnum_t pmap_vtophys(
pmap_t pmap, addr64_t va);
void pmap_switch_user_ttb(
pmap_t pmap);
static void flush_mmu_tlb_region_asid(
vm_offset_t va, unsigned length, pmap_t pmap);
static kern_return_t pmap_expand(
pmap_t, vm_map_address_t, unsigned int options, unsigned int level);
static int pmap_remove_range(
pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *);
static int pmap_remove_range_options(
pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *, int);
static tt_entry_t *pmap_tt1_allocate(
pmap_t, vm_size_t, unsigned int);
#define PMAP_TT_ALLOCATE_NOWAIT 0x1
static void pmap_tt1_deallocate(
pmap_t, tt_entry_t *, vm_size_t, unsigned int);
#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1
static kern_return_t pmap_tt_allocate(
pmap_t, tt_entry_t **, unsigned int, unsigned int);
#define PMAP_TT_ALLOCATE_NOWAIT 0x1
static void pmap_tte_deallocate(
pmap_t, tt_entry_t *, unsigned int);
#define PMAP_TT_L1_LEVEL 0x1
#define PMAP_TT_L2_LEVEL 0x2
#define PMAP_TT_L3_LEVEL 0x3
#if (__ARM_VMSA__ == 7)
#define PMAP_TT_MAX_LEVEL PMAP_TT_L2_LEVEL
#else
#define PMAP_TT_MAX_LEVEL PMAP_TT_L3_LEVEL
#endif
#ifdef __ARM64_PMAP_SUBPAGE_L1__
#if (__ARM_VMSA__ <= 7)
#error This is not supported for old-style page tables
#endif
#define PMAP_ROOT_ALLOC_SIZE (((ARM_TT_L1_INDEX_MASK >> ARM_TT_L1_SHIFT) + 1) * sizeof(tt_entry_t))
#else
#define PMAP_ROOT_ALLOC_SIZE (ARM_PGBYTES)
#endif
const unsigned int arm_hardware_page_size = ARM_PGBYTES;
const unsigned int arm_pt_desc_size = sizeof(pt_desc_t);
const unsigned int arm_pt_root_size = PMAP_ROOT_ALLOC_SIZE;
#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1
void pmap_init_pte_page_internal(
pmap_t, pt_entry_t *, vm_offset_t, unsigned int , pt_desc_t **);
#if (__ARM_VMSA__ > 7)
static inline tt_entry_t *pmap_tt1e(
pmap_t, vm_map_address_t);
static inline tt_entry_t *pmap_tt2e(
pmap_t, vm_map_address_t);
static inline pt_entry_t *pmap_tt3e(
pmap_t, vm_map_address_t);
static void pmap_unmap_sharedpage(
pmap_t pmap);
static void pmap_sharedpage_flush_32_to_64(
void);
static boolean_t
pmap_is_64bit(pmap_t);
#endif
static inline tt_entry_t *pmap_tte(
pmap_t, vm_map_address_t);
static inline pt_entry_t *pmap_pte(
pmap_t, vm_map_address_t);
static void pmap_update_cache_attributes_locked(
ppnum_t, unsigned);
boolean_t arm_clear_fast_fault(
ppnum_t ppnum,
vm_prot_t fault_type);
static pmap_paddr_t pmap_pages_reclaim(
void);
static kern_return_t pmap_pages_alloc(
pmap_paddr_t *pa,
unsigned size,
unsigned option);
#define PMAP_PAGES_ALLOCATE_NOWAIT 0x1
#define PMAP_PAGES_RECLAIM_NOWAIT 0x2
static void pmap_pages_free(
pmap_paddr_t pa,
unsigned size);
#define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \
static __return_type __function_name##_internal __function_args;
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
arm_fast_fault, (pmap_t pmap,
vm_map_address_t va,
vm_prot_t fault_type,
boolean_t from_user), ARM_FAST_FAULT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
arm_force_fast_fault, (ppnum_t ppnum,
vm_prot_t allow_mode,
int options), ARM_FORCE_FAST_FAULT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
mapping_free_prime, (void), MAPPING_FREE_PRIME_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
mapping_replenish, (void), MAPPING_REPLENISH_INDEX);
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
pmap_batch_set_cache_attributes, (ppnum_t pn,
unsigned int cacheattr,
unsigned int page_cnt,
unsigned int page_index,
boolean_t doit,
unsigned int *res), PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_change_wiring, (pmap_t pmap,
vm_map_address_t v,
boolean_t wired), PMAP_CHANGE_WIRING_INDEX);
PMAP_SUPPORT_PROTOTYPES(
pmap_t,
pmap_create, (ledger_t ledger,
vm_map_size_t size,
boolean_t is_64bit), PMAP_CREATE_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_destroy, (pmap_t pmap), PMAP_DESTROY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_enter_options, (pmap_t pmap,
vm_map_address_t v,
ppnum_t pn,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options), PMAP_ENTER_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
vm_offset_t,
pmap_extract, (pmap_t pmap,
vm_map_address_t va), PMAP_EXTRACT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
ppnum_t,
pmap_find_phys, (pmap_t pmap,
addr64_t va), PMAP_FIND_PHYS_INDEX);
#if (__ARM_VMSA__ > 7)
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_insert_sharedpage, (pmap_t pmap), PMAP_INSERT_SHAREDPAGE_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
pmap_is_empty, (pmap_t pmap,
vm_map_offset_t va_start,
vm_map_offset_t va_end), PMAP_IS_EMPTY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
unsigned int,
pmap_map_cpu_windows_copy, (ppnum_t pn,
vm_prot_t prot,
unsigned int wimg_bits), PMAP_MAP_CPU_WINDOWS_COPY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_nest, (pmap_t grand,
pmap_t subord,
addr64_t vstart,
addr64_t nstart,
uint64_t size), PMAP_NEST_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_page_protect_options, (ppnum_t ppnum,
vm_prot_t prot,
unsigned int options), PMAP_PAGE_PROTECT_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_protect_options, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
vm_prot_t prot,
unsigned int options,
void *args), PMAP_PROTECT_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_query_page_info, (pmap_t pmap,
vm_map_offset_t va,
int *disp_p), PMAP_QUERY_PAGE_INFO_INDEX);
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
pmap_query_resident, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
mach_vm_size_t *resident_bytes_p,
mach_vm_size_t *compressed_bytes_p), PMAP_QUERY_RESIDENT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_reference, (pmap_t pmap), PMAP_REFERENCE_INDEX);
PMAP_SUPPORT_PROTOTYPES(
int,
pmap_remove_options, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
int options), PMAP_REMOVE_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_return, (boolean_t do_panic,
boolean_t do_recurse), PMAP_RETURN_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_cache_attributes, (ppnum_t pn,
unsigned int cacheattr), PMAP_SET_CACHE_ATTRIBUTES_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_nested, (pmap_t pmap), PMAP_SET_NESTED_INDEX);
#if MACH_ASSERT
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_process, (pmap_t pmap,
int pid,
char *procname), PMAP_SET_PROCESS_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_unmap_cpu_windows_copy, (unsigned int index), PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_unnest_options, (pmap_t grand,
addr64_t vaddr,
uint64_t size,
unsigned int option), PMAP_UNNEST_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
phys_attribute_set, (ppnum_t pn,
unsigned int bits), PHYS_ATTRIBUTE_SET_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
phys_attribute_clear, (ppnum_t pn,
unsigned int bits,
int options,
void *arg), PHYS_ATTRIBUTE_CLEAR_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_switch, (pmap_t pmap), PMAP_SWITCH_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_switch_user_ttb, (pmap_t pmap), PMAP_SWITCH_USER_TTB_INDEX);
void pmap_footprint_suspend(vm_map_t map,
boolean_t suspend);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_footprint_suspend, (vm_map_t map,
boolean_t suspend),
PMAP_FOOTPRINT_SUSPEND_INDEX);
#if CONFIG_PGTRACE
boolean_t pgtrace_enabled = 0;
typedef struct {
queue_chain_t chain;
pmap_t pmap;
vm_map_offset_t ova;
vm_map_offset_t cva[3];
pt_entry_t cva_spte[3];
struct {
pmap_paddr_t start;
pmap_paddr_t end;
} range;
bool cloned;
} pmap_pgtrace_map_t;
static void pmap_pgtrace_init(void);
static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end);
static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa_page, vm_map_offset_t va_page);
static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa);
#endif
#if (__ARM_VMSA__ > 7)
#if (ARM_PGSHIFT == 14)
#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x4000)
#else
#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x2000)
#endif
#else
#define LOWGLOBAL_ALIAS (0xFFFF1000)
#endif
long long alloc_tteroot_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
long long alloc_ttepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
long long alloc_ptepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
long long alloc_pmap_pages_count __attribute__((aligned(8))) = 0LL;
int pt_fake_zone_index = -1;
static void
pmap_cpu_data_init_internal(unsigned int cpu_number)
{
pmap_cpu_data_t * pmap_cpu_data = NULL;
pmap_cpu_data = pmap_get_cpu_data();
pmap_cpu_data->cpu_number = cpu_number;
}
void
pmap_cpu_data_init(void)
{
pmap_cpu_data_init_internal(cpu_number());
}
static void
pmap_cpu_data_array_init(void)
{
pmap_cpu_data_init();
}
pmap_cpu_data_t *
pmap_get_cpu_data(void)
{
pmap_cpu_data_t * pmap_cpu_data = NULL;
pmap_cpu_data = &getCpuDatap()->cpu_pmap_cpu_data;
return pmap_cpu_data;
}
pmap_paddr_t
pmap_pages_reclaim(
void)
{
boolean_t found_page;
unsigned i;
pt_desc_t *ptdp;
simple_lock(&pmap_pages_lock);
pmap_pages_request_count++;
pmap_pages_request_acum++;
while (1) {
if (pmap_pages_reclaim_list != (page_free_entry_t *)NULL) {
page_free_entry_t *page_entry;
page_entry = pmap_pages_reclaim_list;
pmap_pages_reclaim_list = pmap_pages_reclaim_list->next;
simple_unlock(&pmap_pages_lock);
return((pmap_paddr_t)ml_static_vtop((vm_offset_t)page_entry));
}
simple_unlock(&pmap_pages_lock);
simple_lock(&pt_pages_lock);
ptdp = (pt_desc_t *)queue_first(&pt_page_list);
found_page = FALSE;
while (!queue_end(&pt_page_list, (queue_entry_t)ptdp)) {
if ((ptdp->pmap != kernel_pmap)
&& (ptdp->pmap->nested == FALSE)
&& (simple_lock_try(&ptdp->pmap->lock))) {
unsigned refcnt_acc = 0;
unsigned wiredcnt_acc = 0;
for (i = 0 ; i < PT_INDEX_MAX ; i++) {
if (ptdp->pt_cnt[i].refcnt & PT_DESC_REFCOUNT) {
refcnt_acc = 0;
break;
}
refcnt_acc += ptdp->pt_cnt[i].refcnt;
wiredcnt_acc += ptdp->pt_cnt[i].wiredcnt;
}
if ((wiredcnt_acc == 0) && (refcnt_acc != 0)) {
found_page = TRUE;
break;
}
simple_unlock(&ptdp->pmap->lock);
}
ptdp = (pt_desc_t *)queue_next((queue_t)ptdp);
}
if (!found_page) {
panic("pmap_pages_reclaim(): No eligible page in pt_page_list\n");
} else {
int remove_count = 0;
vm_map_address_t va;
pmap_t pmap;
pt_entry_t *bpte, *epte;
pt_entry_t *pte_p;
tt_entry_t *tte_p;
uint32_t rmv_spte=0;
simple_unlock(&pt_pages_lock);
pmap = ptdp->pmap;
PMAP_ASSERT_LOCKED(pmap); for (i = 0 ; i < PT_INDEX_MAX ; i++) {
va = ptdp->pt_map[i].va;
tte_p = pmap_tte(pmap, va);
if ((tte_p != (tt_entry_t *) NULL)
&& ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) {
#if (__ARM_VMSA__ == 7)
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[ptenum(va)];
epte = bpte + PAGE_SIZE/sizeof(pt_entry_t);
#else
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[tt3_index(pmap, va)];
epte = bpte + PAGE_SIZE/sizeof(pt_entry_t);
#endif
remove_count += pmap_remove_range_options(
pmap, va, bpte, epte,
&rmv_spte, PMAP_OPTIONS_REMOVE);
if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt != 0)
panic("pmap_pages_reclaim(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt);
#if (__ARM_VMSA__ == 7)
pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL);
flush_mmu_tlb_entry((va & ~ARM_TT_L1_PT_OFFMASK) | (pmap->asid & 0xff));
flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + ARM_TT_L1_SIZE) | (pmap->asid & 0xff));
flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 2*ARM_TT_L1_SIZE)| (pmap->asid & 0xff));
flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 3*ARM_TT_L1_SIZE)| (pmap->asid & 0xff));
#else
pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL);
flush_mmu_tlb_entry(tlbi_addr(va & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
#endif
if (remove_count > 0) {
#if (__ARM_VMSA__ == 7)
PMAP_UPDATE_TLBS(pmap, va, va+4*ARM_TT_L1_SIZE);
#else
PMAP_UPDATE_TLBS(pmap, va, va+ARM_TT_L2_SIZE);
#endif
}
}
}
PMAP_UNLOCK(pmap);
}
simple_lock(&pmap_pages_lock);
}
}
static kern_return_t
pmap_pages_alloc(
pmap_paddr_t *pa,
unsigned size,
unsigned option)
{
vm_page_t m = VM_PAGE_NULL, m_prev;
if(option & PMAP_PAGES_RECLAIM_NOWAIT) {
assert(size == PAGE_SIZE);
*pa = pmap_pages_reclaim();
return KERN_SUCCESS;
}
if (size == PAGE_SIZE) {
while ((m = vm_page_grab()) == VM_PAGE_NULL) {
if(option & PMAP_PAGES_ALLOCATE_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
vm_page_lock_queues();
vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
vm_page_unlock_queues();
}
if (size == 2*PAGE_SIZE) {
while (cpm_allocate(size, &m, 0, 1, TRUE, 0) != KERN_SUCCESS) {
if(option & PMAP_PAGES_ALLOCATE_NOWAIT)
return KERN_RESOURCE_SHORTAGE;
VM_PAGE_WAIT();
}
}
*pa = (pmap_paddr_t)ptoa(VM_PAGE_GET_PHYS_PAGE(m));
vm_object_lock(pmap_object);
while (m != VM_PAGE_NULL) {
vm_page_insert_wired(m, pmap_object, (vm_object_offset_t) ((ptoa(VM_PAGE_GET_PHYS_PAGE(m))) - gPhysBase), VM_KERN_MEMORY_PTE);
m_prev = m;
m = NEXT_PAGE(m_prev);
*(NEXT_PAGE_PTR(m_prev)) = VM_PAGE_NULL;
}
vm_object_unlock(pmap_object);
OSAddAtomic(size>>PAGE_SHIFT, &inuse_pmap_pages_count);
OSAddAtomic64(size>>PAGE_SHIFT, &alloc_pmap_pages_count);
return KERN_SUCCESS;
}
static void
pmap_pages_free(
pmap_paddr_t pa,
unsigned size)
{
simple_lock(&pmap_pages_lock);
if (pmap_pages_request_count != 0) {
page_free_entry_t *page_entry;
pmap_pages_request_count--;
page_entry = (page_free_entry_t *)phystokv(pa);
page_entry->next = pmap_pages_reclaim_list;
pmap_pages_reclaim_list = page_entry;
simple_unlock(&pmap_pages_lock);
return;
}
simple_unlock(&pmap_pages_lock);
vm_page_t m;
pmap_paddr_t pa_max;
OSAddAtomic(-(size>>PAGE_SHIFT), &inuse_pmap_pages_count);
for (pa_max = pa + size; pa < pa_max; pa = pa + PAGE_SIZE) {
vm_object_lock(pmap_object);
m = vm_page_lookup(pmap_object, (pa - gPhysBase));
assert(m != VM_PAGE_NULL);
assert(VM_PAGE_WIRED(m));
vm_page_lock_queues();
vm_page_free(m);
vm_page_unlock_queues();
vm_object_unlock(pmap_object);
}
}
static inline void
PMAP_ZINFO_PALLOC(
pmap_t pmap, int bytes)
{
pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes);
}
static inline void
PMAP_ZINFO_PFREE(
pmap_t pmap,
int bytes)
{
pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes);
}
static inline void
pmap_tt_ledger_credit(
pmap_t pmap,
vm_size_t size)
{
if (pmap != kernel_pmap) {
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, size);
pmap_ledger_credit(pmap, task_ledgers.page_table, size);
}
}
static inline void
pmap_tt_ledger_debit(
pmap_t pmap,
vm_size_t size)
{
if (pmap != kernel_pmap) {
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, size);
pmap_ledger_debit(pmap, task_ledgers.page_table, size);
}
}
static unsigned int
alloc_asid(
void)
{
unsigned int asid_bitmap_index;
simple_lock(&pmaps_lock);
for (asid_bitmap_index = 0; asid_bitmap_index < (MAX_ASID / (sizeof(uint32_t) * NBBY)); asid_bitmap_index++) {
unsigned int temp = ffs(asid_bitmap[asid_bitmap_index]);
if (temp > 0) {
temp -= 1;
asid_bitmap[asid_bitmap_index] &= ~(1 << temp);
#if __ARM_KERNEL_PROTECT__
unsigned int temp2 = temp | 1;
assert(temp2 < MAX_ASID);
assert(temp2 < 32);
assert(temp2 != temp);
assert(asid_bitmap[asid_bitmap_index] & (1 << temp2));
asid_bitmap[asid_bitmap_index] &= ~(1 << temp2);
#endif
simple_unlock(&pmaps_lock);
assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0);
#if __ARM_KERNEL_PROTECT__
assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 1);
#endif
return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp);
}
}
simple_unlock(&pmaps_lock);
panic("alloc_asid(): out of ASID number");
return MAX_ASID;
}
static void
free_asid(
int asid)
{
assert((asid % ARM_MAX_ASID) != 0);
simple_lock(&pmaps_lock);
setbit(asid, (int *) asid_bitmap);
#if __ARM_KERNEL_PROTECT__
assert((asid | 1) < MAX_ASID);
assert((asid | 1) != asid);
setbit(asid | 1, (int *) asid_bitmap);
#endif
simple_unlock(&pmaps_lock);
}
#define PV_LOW_WATER_MARK_DEFAULT 0x200
#define PV_KERN_LOW_WATER_MARK_DEFAULT 0x200
#define PV_ALLOC_CHUNK_INITIAL 0x200
#define PV_KERN_ALLOC_CHUNK_INITIAL 0x200
#define PV_ALLOC_INITIAL_TARGET (PV_ALLOC_CHUNK_INITIAL * 5)
#define PV_KERN_ALLOC_INITIAL_TARGET (PV_KERN_ALLOC_CHUNK_INITIAL)
uint32_t pv_free_count MARK_AS_PMAP_DATA = 0;
uint32_t pv_page_count MARK_AS_PMAP_DATA = 0;
uint32_t pv_kern_free_count MARK_AS_PMAP_DATA = 0;
uint32_t pv_low_water_mark MARK_AS_PMAP_DATA;
uint32_t pv_kern_low_water_mark MARK_AS_PMAP_DATA;
uint32_t pv_alloc_chunk MARK_AS_PMAP_DATA;
uint32_t pv_kern_alloc_chunk MARK_AS_PMAP_DATA;
thread_t mapping_replenish_thread;
event_t mapping_replenish_event;
event_t pmap_user_pv_throttle_event;
volatile uint32_t mappingrecurse = 0;
uint64_t pmap_pv_throttle_stat;
uint64_t pmap_pv_throttled_waiters;
unsigned pmap_mapping_thread_wakeups;
unsigned pmap_kernel_reserve_replenish_stat MARK_AS_PMAP_DATA;
unsigned pmap_user_reserve_replenish_stat MARK_AS_PMAP_DATA;
unsigned pmap_kern_reserve_alloc_stat MARK_AS_PMAP_DATA;
static void
pv_init(
void)
{
simple_lock_init(&pv_free_list_lock, 0);
simple_lock_init(&pv_kern_free_list_lock, 0);
pv_free_list = PV_ENTRY_NULL;
pv_free_count = 0x0U;
pv_kern_free_list = PV_ENTRY_NULL;
pv_kern_free_count = 0x0U;
}
static inline void PV_ALLOC(pv_entry_t **pv_ep);
static inline void PV_KERN_ALLOC(pv_entry_t **pv_e);
static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt);
static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt);
static inline void pmap_pv_throttle(pmap_t p);
static boolean_t
pv_alloc(
pmap_t pmap,
unsigned int pai,
pv_entry_t **pvepp)
{
PMAP_ASSERT_LOCKED(pmap);
ASSERT_PVH_LOCKED(pai);
PV_ALLOC(pvepp);
if (PV_ENTRY_NULL == *pvepp) {
if (kernel_pmap == pmap) {
PV_KERN_ALLOC(pvepp);
if (PV_ENTRY_NULL == *pvepp) {
pv_entry_t *pv_e;
pv_entry_t *pv_eh;
pv_entry_t *pv_et;
int pv_cnt;
unsigned j;
pmap_paddr_t pa;
kern_return_t ret;
UNLOCK_PVH(pai);
PMAP_UNLOCK(pmap);
ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT);
if (ret == KERN_RESOURCE_SHORTAGE) {
ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT);
}
if (ret != KERN_SUCCESS) {
panic("%s: failed to alloc page for kernel, ret=%d, "
"pmap=%p, pai=%u, pvepp=%p",
__FUNCTION__, ret,
pmap, pai, pvepp);
}
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
pv_cnt = 0;
pv_eh = pv_et = PV_ENTRY_NULL;
*pvepp = pv_e;
pv_e++;
for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
pv_e->pve_next = pv_eh;
pv_eh = pv_e;
if (pv_et == PV_ENTRY_NULL)
pv_et = pv_e;
pv_cnt++;
pv_e++;
}
PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt);
PMAP_LOCK(pmap);
LOCK_PVH(pai);
return FALSE;
}
} else {
UNLOCK_PVH(pai);
PMAP_UNLOCK(pmap);
pmap_pv_throttle(pmap);
{
pv_entry_t *pv_e;
pv_entry_t *pv_eh;
pv_entry_t *pv_et;
int pv_cnt;
unsigned j;
pmap_paddr_t pa;
kern_return_t ret;
ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0);
if (ret != KERN_SUCCESS) {
panic("%s: failed to alloc page, ret=%d, "
"pmap=%p, pai=%u, pvepp=%p",
__FUNCTION__, ret,
pmap, pai, pvepp);
}
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
pv_cnt = 0;
pv_eh = pv_et = PV_ENTRY_NULL;
*pvepp = pv_e;
pv_e++;
for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
pv_e->pve_next = pv_eh;
pv_eh = pv_e;
if (pv_et == PV_ENTRY_NULL)
pv_et = pv_e;
pv_cnt++;
pv_e++;
}
PV_FREE_LIST(pv_eh, pv_et, pv_cnt);
}
PMAP_LOCK(pmap);
LOCK_PVH(pai);
return FALSE;
}
}
assert(PV_ENTRY_NULL != *pvepp);
return TRUE;
}
static void
pv_free(
pv_entry_t *pvep)
{
PV_FREE_LIST(pvep, pvep, 1);
}
static void
pv_list_free(
pv_entry_t *pvehp,
pv_entry_t *pvetp,
unsigned int cnt)
{
PV_FREE_LIST(pvehp, pvetp, cnt);
}
static inline void PV_ALLOC(pv_entry_t **pv_ep) {
assert(*pv_ep == PV_ENTRY_NULL);
simple_lock(&pv_free_list_lock);
if ((pv_kern_free_count >= pv_kern_low_water_mark) && ((*pv_ep = pv_free_list) != 0)) {
pv_free_list = (pv_entry_t *)(*pv_ep)->pve_next;
(*pv_ep)->pve_next = PV_ENTRY_NULL;
pv_free_count--;
}
simple_unlock(&pv_free_list_lock);
if ((pv_free_count < pv_low_water_mark) || (pv_kern_free_count < pv_kern_low_water_mark)) {
if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse))
thread_wakeup(&mapping_replenish_event);
}
}
static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) {
simple_lock(&pv_free_list_lock);
pv_et->pve_next = (pv_entry_t *)pv_free_list;
pv_free_list = pv_eh;
pv_free_count += pv_cnt;
simple_unlock(&pv_free_list_lock);
}
static inline void PV_KERN_ALLOC(pv_entry_t **pv_e) {
assert(*pv_e == PV_ENTRY_NULL);
simple_lock(&pv_kern_free_list_lock);
if ((*pv_e = pv_kern_free_list) != 0) {
pv_kern_free_list = (pv_entry_t *)(*pv_e)->pve_next;
(*pv_e)->pve_next = PV_ENTRY_NULL;
pv_kern_free_count--;
pmap_kern_reserve_alloc_stat++;
}
simple_unlock(&pv_kern_free_list_lock);
if (pv_kern_free_count < pv_kern_low_water_mark) {
if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse)) {
thread_wakeup(&mapping_replenish_event);
}
}
}
static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) {
simple_lock(&pv_kern_free_list_lock);
pv_et->pve_next = pv_kern_free_list;
pv_kern_free_list = pv_eh;
pv_kern_free_count += pv_cnt;
simple_unlock(&pv_kern_free_list_lock);
}
static inline void pmap_pv_throttle(__unused pmap_t p) {
assert(p != kernel_pmap);
if (pv_kern_free_count < (pv_kern_low_water_mark / 2)) {
pmap_pv_throttle_stat++;
pmap_pv_throttled_waiters++;
assert_wait_timeout(&pmap_user_pv_throttle_event, THREAD_UNINT, 1, 1000 * NSEC_PER_USEC);
thread_block(THREAD_CONTINUE_NULL);
}
}
static kern_return_t
mapping_free_prime_internal(void)
{
unsigned j;
pmap_paddr_t pa;
kern_return_t ret;
pv_entry_t *pv_e;
pv_entry_t *pv_eh;
pv_entry_t *pv_et;
int pv_cnt;
int alloc_options = 0;
int needed_pv_cnt = 0;
int target_pv_free_cnt = 0;
SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_called = FALSE;
SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_done = FALSE;
if (mapping_free_prime_internal_done) {
return KERN_FAILURE;
}
if (!mapping_free_prime_internal_called) {
mapping_free_prime_internal_called = TRUE;
pv_low_water_mark = PV_LOW_WATER_MARK_DEFAULT;
pv_kern_low_water_mark = PV_KERN_LOW_WATER_MARK_DEFAULT;
pv_kern_alloc_chunk = PV_KERN_ALLOC_CHUNK_INITIAL;
pv_alloc_chunk = PV_ALLOC_CHUNK_INITIAL;
}
pv_cnt = 0;
pv_eh = pv_et = PV_ENTRY_NULL;
target_pv_free_cnt = PV_ALLOC_INITIAL_TARGET;
needed_pv_cnt = target_pv_free_cnt - pv_free_count;
if (needed_pv_cnt > target_pv_free_cnt) {
needed_pv_cnt = 0;
}
while (pv_cnt < needed_pv_cnt) {
ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options);
assert(ret == KERN_SUCCESS);
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
pv_e->pve_next = pv_eh;
pv_eh = pv_e;
if (pv_et == PV_ENTRY_NULL)
pv_et = pv_e;
pv_cnt++;
pv_e++;
}
}
if (pv_cnt) {
PV_FREE_LIST(pv_eh, pv_et, pv_cnt);
}
pv_cnt = 0;
pv_eh = pv_et = PV_ENTRY_NULL;
target_pv_free_cnt = PV_KERN_ALLOC_INITIAL_TARGET;
needed_pv_cnt = target_pv_free_cnt - pv_kern_free_count;
if (needed_pv_cnt > target_pv_free_cnt) {
needed_pv_cnt = 0;
}
while (pv_cnt < needed_pv_cnt) {
ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options);
assert(ret == KERN_SUCCESS);
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
pv_e->pve_next = pv_eh;
pv_eh = pv_e;
if (pv_et == PV_ENTRY_NULL)
pv_et = pv_e;
pv_cnt++;
pv_e++;
}
}
if (pv_cnt) {
PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt);
}
mapping_free_prime_internal_done = TRUE;
return KERN_SUCCESS;
}
void
mapping_free_prime(void)
{
kern_return_t kr = KERN_FAILURE;
kr = mapping_free_prime_internal();
if (kr != KERN_SUCCESS) {
panic("%s: failed, kr=%d", __FUNCTION__, kr);
}
}
void mapping_replenish(void);
void mapping_adjust(void) {
kern_return_t mres;
mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread);
if (mres != KERN_SUCCESS) {
panic("pmap: mapping_replenish thread creation failed");
}
thread_deallocate(mapping_replenish_thread);
}
static kern_return_t
mapping_replenish_internal(void)
{
pv_entry_t *pv_e;
pv_entry_t *pv_eh;
pv_entry_t *pv_et;
int pv_cnt;
unsigned j;
pmap_paddr_t pa;
kern_return_t ret = KERN_SUCCESS;
while (pv_kern_free_count < pv_kern_low_water_mark) {
pv_cnt = 0;
pv_eh = pv_et = PV_ENTRY_NULL;
ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0);
assert(ret == KERN_SUCCESS);
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
pv_e->pve_next = pv_eh;
pv_eh = pv_e;
if (pv_et == PV_ENTRY_NULL)
pv_et = pv_e;
pv_cnt++;
pv_e++;
}
pmap_kernel_reserve_replenish_stat += pv_cnt;
PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt);
}
while (pv_free_count < pv_low_water_mark) {
pv_cnt = 0;
pv_eh = pv_et = PV_ENTRY_NULL;
ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0);
assert(ret == KERN_SUCCESS);
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) {
pv_e->pve_next = pv_eh;
pv_eh = pv_e;
if (pv_et == PV_ENTRY_NULL)
pv_et = pv_e;
pv_cnt++;
pv_e++;
}
pmap_user_reserve_replenish_stat += pv_cnt;
PV_FREE_LIST(pv_eh, pv_et, pv_cnt);
}
return ret;
}
__attribute__((noreturn))
void
mapping_replenish(void)
{
kern_return_t kr;
current_thread()->options |= TH_OPT_VMPRIV;
for (;;) {
kr = mapping_replenish_internal();
if (kr != KERN_SUCCESS) {
panic("%s: failed, kr=%d", __FUNCTION__, kr);
}
if (pmap_pv_throttled_waiters) {
pmap_pv_throttled_waiters = 0;
thread_wakeup(&pmap_user_pv_throttle_event);
}
if (pv_kern_free_count < pv_kern_low_water_mark)
continue;
assert_wait(&mapping_replenish_event, THREAD_UNINT);
mappingrecurse = 0;
thread_block(THREAD_CONTINUE_NULL);
pmap_mapping_thread_wakeups++;
}
}
static void
ptd_bootstrap(
pt_desc_t *ptdp,
unsigned int ptd_cnt)
{
simple_lock_init(&ptd_free_list_lock, 0);
while (ptd_cnt != 0) {
(*(void **)ptdp) = (void *)ptd_free_list;
ptd_free_list = ptdp;
ptdp++;
ptd_cnt--;
ptd_free_count++;
}
ptd_preboot = FALSE;
}
static pt_desc_t
*ptd_alloc(
pmap_t pmap)
{
pt_desc_t *ptdp;
unsigned i;
if (!ptd_preboot)
simple_lock(&ptd_free_list_lock);
if (ptd_free_count == 0) {
unsigned int ptd_cnt;
pt_desc_t *ptdp_next;
if (ptd_preboot) {
ptdp = (pt_desc_t *)avail_start;
avail_start += ARM_PGBYTES;
ptdp_next = ptdp;
ptd_cnt = ARM_PGBYTES/sizeof(pt_desc_t);
} else {
pmap_paddr_t pa;
kern_return_t ret;
simple_unlock(&ptd_free_list_lock);
if (pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT) != KERN_SUCCESS) {
ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT);
assert(ret == KERN_SUCCESS);
}
ptdp = (pt_desc_t *)phystokv(pa);
simple_lock(&ptd_free_list_lock);
ptdp_next = ptdp;
ptd_cnt = PAGE_SIZE/sizeof(pt_desc_t);
}
while (ptd_cnt != 0) {
(*(void **)ptdp_next) = (void *)ptd_free_list;
ptd_free_list = ptdp_next;
ptdp_next++;
ptd_cnt--;
ptd_free_count++;
}
}
if ((ptdp = ptd_free_list) != PTD_ENTRY_NULL) {
ptd_free_list = (pt_desc_t *)(*(void **)ptdp);
ptd_free_count--;
} else {
panic("out of ptd entry\n");
}
if (!ptd_preboot)
simple_unlock(&ptd_free_list_lock);
ptdp->pt_page.next = NULL;
ptdp->pt_page.prev = NULL;
ptdp->pmap = pmap;
for (i = 0 ; i < PT_INDEX_MAX ; i++) {
ptdp->pt_map[i].va = 0;
ptdp->pt_cnt[i].refcnt = 0;
ptdp->pt_cnt[i].wiredcnt = 0;
}
simple_lock(&pt_pages_lock);
queue_enter(&pt_page_list, ptdp, pt_desc_t *, pt_page);
simple_unlock(&pt_pages_lock);
pmap_tt_ledger_credit(pmap, sizeof(*ptdp));
return(ptdp);
}
static void
ptd_deallocate(
pt_desc_t *ptdp)
{
unsigned i;
pmap_t pmap = ptdp->pmap;
if (ptd_preboot) {
panic("ptd_deallocate(): early boot\n");
}
for (i = 0 ; i < PT_INDEX_MAX ; i++) {
if (ptdp->pt_cnt[i].refcnt != 0)
panic("ptd_deallocate(): ptdp=%p refcnt=0x%x \n", ptdp, ptdp->pt_cnt[i].refcnt);
}
if (ptdp->pt_page.next != NULL) {
simple_lock(&pt_pages_lock);
queue_remove(&pt_page_list, ptdp, pt_desc_t *, pt_page);
simple_unlock(&pt_pages_lock);
}
simple_lock(&ptd_free_list_lock);
(*(void **)ptdp) = (void *)ptd_free_list;
ptd_free_list = (pt_desc_t *)ptdp;
ptd_free_count++;
simple_unlock(&ptd_free_list_lock);
pmap_tt_ledger_debit(pmap, sizeof(*ptdp));
}
static void
ptd_init(
pt_desc_t *ptdp,
pmap_t pmap,
vm_map_address_t va,
unsigned int level,
pt_entry_t *pte_p)
{
if (ptdp->pmap != pmap)
panic("ptd_init(): pmap mismatch\n");
#if (__ARM_VMSA__ == 7)
assert(level == 2);
ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~(ARM_TT_L1_PT_OFFMASK);
#else
if (level == 3) {
ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L2_OFFMASK ;
} else if (level == 2)
ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L1_OFFMASK ;
#endif
if (level < PMAP_TT_MAX_LEVEL)
ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt = PT_DESC_REFCOUNT;
}
boolean_t
pmap_valid_address(
pmap_paddr_t addr)
{
return pa_valid(addr);
}
#if (__ARM_VMSA__ == 7)
static inline tt_entry_t *
pmap_tte(pmap_t pmap,
vm_map_address_t addr)
{
if (!(tte_index(pmap, addr) < pmap->tte_index_max))
return (tt_entry_t *)NULL;
return (&pmap->tte[tte_index(pmap, addr)]);
}
static inline pt_entry_t *
pmap_pte(
pmap_t pmap,
vm_map_address_t addr)
{
pt_entry_t *ptp;
tt_entry_t *ttp;
tt_entry_t tte;
ttp = pmap_tte(pmap, addr);
if (ttp == (tt_entry_t *)NULL)
return (PT_ENTRY_NULL);
tte = *ttp;
#if MACH_ASSERT
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK)
panic("Attempt to demote L1 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
#endif
if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)
return (PT_ENTRY_NULL);
ptp = (pt_entry_t *) ttetokv(tte) + ptenum(addr);
return (ptp);
}
#else
static inline tt_entry_t *
pmap_tt1e(pmap_t pmap,
vm_map_address_t addr)
{
#if __ARM64_TWO_LEVEL_PMAP__
#pragma unused(pmap, addr)
panic("pmap_tt1e called on a two level pmap");
return (NULL);
#else
return (&pmap->tte[tt1_index(pmap, addr)]);
#endif
}
static inline tt_entry_t *
pmap_tt2e(pmap_t pmap,
vm_map_address_t addr)
{
#if __ARM64_TWO_LEVEL_PMAP__
return (&pmap->tte[tt2_index(pmap, addr)]);
#else
tt_entry_t *ttp;
tt_entry_t tte;
ttp = pmap_tt1e(pmap, addr);
tte = *ttp;
#if MACH_ASSERT
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID))
panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
#endif
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID))
return (PT_ENTRY_NULL);
ttp = &((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)];
return ((tt_entry_t *)ttp);
#endif
}
static inline pt_entry_t *
pmap_tt3e(
pmap_t pmap,
vm_map_address_t addr)
{
pt_entry_t *ptp;
tt_entry_t *ttp;
tt_entry_t tte;
#if __ARM64_TWO_LEVEL_PMAP__
ttp = pmap_tt2e(pmap, addr);
tte = *ttp;
#else
ttp = pmap_tt1e(pmap, addr);
tte = *ttp;
#if MACH_ASSERT
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID))
panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
#endif
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID))
return (PT_ENTRY_NULL);
tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)];
#endif
#if MACH_ASSERT
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID))
panic("Attempt to demote L2 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte);
#endif
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
return (PT_ENTRY_NULL);
}
ptp = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, addr)]);
return (ptp);
}
static inline tt_entry_t *
pmap_tte(
pmap_t pmap,
vm_map_address_t addr)
{
return(pmap_tt2e(pmap, addr));
}
static inline pt_entry_t *
pmap_pte(
pmap_t pmap,
vm_map_address_t addr)
{
return(pmap_tt3e(pmap, addr));
}
#endif
vm_map_address_t
pmap_map(
vm_map_address_t virt,
vm_offset_t start,
vm_offset_t end,
vm_prot_t prot,
unsigned int flags)
{
kern_return_t kr;
vm_size_t ps;
ps = PAGE_SIZE;
while (start < end) {
kr = pmap_enter(kernel_pmap, virt, (ppnum_t)atop(start),
prot, VM_PROT_NONE, flags, FALSE);
if (kr != KERN_SUCCESS) {
panic("%s: failed pmap_enter, "
"virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x",
__FUNCTION__,
(void *) virt, (void *) start, (void *) end, prot, flags);
}
virt += ps;
start += ps;
}
return (virt);
}
vm_map_address_t
pmap_map_bd_with_options(
vm_map_address_t virt,
vm_offset_t start,
vm_offset_t end,
vm_prot_t prot,
int32_t options)
{
pt_entry_t tmplate;
pt_entry_t *ptep;
vm_map_address_t vaddr;
vm_offset_t paddr;
pt_entry_t mem_attr;
switch (options & PMAP_MAP_BD_MASK) {
case PMAP_MAP_BD_WCOMB:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB);
#if (__ARM_VMSA__ > 7)
mem_attr |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
mem_attr |= ARM_PTE_SH;
#endif
break;
case PMAP_MAP_BD_POSTED:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED);
break;
default:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
break;
}
tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) |
mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF;
#if __ARM_KERNEL_PROTECT__
tmplate |= ARM_PTE_NG;
#endif
vaddr = virt;
paddr = start;
while (paddr < end) {
ptep = pmap_pte(kernel_pmap, vaddr);
if (ptep == PT_ENTRY_NULL) {
panic("pmap_map_bd");
}
assert(!ARM_PTE_IS_COMPRESSED(*ptep));
WRITE_PTE(ptep, tmplate);
pte_increment_pa(tmplate);
vaddr += PAGE_SIZE;
paddr += PAGE_SIZE;
}
if (end >= start)
flush_mmu_tlb_region(virt, (unsigned)(end - start));
return (vaddr);
}
vm_map_address_t
pmap_map_bd(
vm_map_address_t virt,
vm_offset_t start,
vm_offset_t end,
vm_prot_t prot)
{
pt_entry_t tmplate;
pt_entry_t *ptep;
vm_map_address_t vaddr;
vm_offset_t paddr;
tmplate = pa_to_pte(start)
| ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
| ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
#if __ARM_KERNEL_PROTECT__
tmplate |= ARM_PTE_NG;
#endif
vaddr = virt;
paddr = start;
while (paddr < end) {
ptep = pmap_pte(kernel_pmap, vaddr);
if (ptep == PT_ENTRY_NULL) {
panic("pmap_map_bd");
}
assert(!ARM_PTE_IS_COMPRESSED(*ptep));
WRITE_PTE(ptep, tmplate);
pte_increment_pa(tmplate);
vaddr += PAGE_SIZE;
paddr += PAGE_SIZE;
}
if (end >= start)
flush_mmu_tlb_region(virt, (unsigned)(end - start));
return (vaddr);
}
vm_map_address_t
pmap_map_high_window_bd(
vm_offset_t pa_start,
vm_size_t len,
vm_prot_t prot)
{
pt_entry_t *ptep, pte;
#if (__ARM_VMSA__ == 7)
vm_map_address_t va_start = VM_HIGH_KERNEL_WINDOW;
vm_map_address_t va_max = VM_MAX_KERNEL_ADDRESS;
#else
vm_map_address_t va_start = VREGION1_START;
vm_map_address_t va_max = VREGION1_START + VREGION1_SIZE;
#endif
vm_map_address_t va_end;
vm_map_address_t va;
vm_size_t offset;
offset = pa_start & PAGE_MASK;
pa_start -= offset;
len += offset;
if (len > (va_max - va_start)) {
panic("pmap_map_high_window_bd: area too large\n");
}
scan:
for ( ; va_start < va_max; va_start += PAGE_SIZE) {
ptep = pmap_pte(kernel_pmap, va_start);
assert(!ARM_PTE_IS_COMPRESSED(*ptep));
if (*ptep == ARM_PTE_TYPE_FAULT)
break;
}
if (va_start > va_max) {
panic("pmap_map_high_window_bd: insufficient pages\n");
}
for (va_end = va_start + PAGE_SIZE; va_end < va_start + len; va_end += PAGE_SIZE) {
ptep = pmap_pte(kernel_pmap, va_end);
assert(!ARM_PTE_IS_COMPRESSED(*ptep));
if (*ptep != ARM_PTE_TYPE_FAULT) {
va_start = va_end + PAGE_SIZE;
goto scan;
}
}
for (va = va_start; va < va_end; va += PAGE_SIZE, pa_start += PAGE_SIZE) {
ptep = pmap_pte(kernel_pmap, va);
pte = pa_to_pte(pa_start)
| ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
| ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
WRITE_PTE(ptep, pte);
}
PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len);
#if KASAN
kasan_notify_address(va_start, len);
#endif
return va_start;
}
#define PMAP_ALIGN(addr, align) ((addr) + ((align) - 1) & ~((align) - 1))
typedef struct pmap_io_range
{
uint64_t addr;
uint32_t len;
uint32_t wimg;
} __attribute__((packed)) pmap_io_range_t;
static unsigned int
pmap_compute_io_rgns(void)
{
DTEntry entry;
pmap_io_range_t *ranges;
void *prop = NULL;
int err;
unsigned int prop_size;
err = DTLookupEntry(NULL, "/defaults", &entry);
assert(err == kSuccess);
if (kSuccess != DTGetProperty(entry, "pmap-io-granule", &prop, &prop_size))
return 0;
io_rgn_granule = *((uint32_t*)prop);
if (kSuccess != DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size))
return 0;
if ((io_rgn_granule == 0) || (io_rgn_granule & PAGE_MASK))
panic("pmap I/O region granularity is not page-aligned!\n");
ranges = prop;
for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) {
if ((i == 0) || (ranges[i].addr < io_rgn_start))
io_rgn_start = ranges[i].addr;
if ((i == 0) || ((ranges[i].addr + ranges[i].len) > io_rgn_end))
io_rgn_end = ranges[i].addr + ranges[i].len;
}
if (io_rgn_start & PAGE_MASK)
panic("pmap I/O region start is not page-aligned!\n");
if (io_rgn_end & PAGE_MASK)
panic("pmap I/O region end is not page-aligned!\n");
if (((io_rgn_start < gPhysBase) && (io_rgn_end >= gPhysBase)) ||
((io_rgn_start < avail_end) && (io_rgn_end >= avail_end)))
panic("pmap I/O region overlaps physical memory!\n");
return (unsigned int)((io_rgn_end - io_rgn_start) / io_rgn_granule);
}
static void
pmap_load_io_rgns(void)
{
DTEntry entry;
pmap_io_range_t *ranges;
void *prop = NULL;
int err;
unsigned int prop_size;
if (io_rgn_granule == 0)
return;
err = DTLookupEntry(NULL, "/defaults", &entry);
assert(err == kSuccess);
err = DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size);
assert(err == kSuccess);
ranges = prop;
for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) {
if ((ranges[i].addr - io_rgn_start) % io_rgn_granule)
panic("pmap I/O region %d is not aligned to I/O granularity!\n", i);
if (ranges[i].len % io_rgn_granule)
panic("pmap I/O region %d size is not a multiple of I/O granularity!\n", i);
for (uint32_t offs = 0; offs < ranges[i].len; offs += io_rgn_granule) {
io_attr_table[(ranges[i].addr + offs - io_rgn_start) / io_rgn_granule] =
IO_ATTR_WIMG(ranges[i].wimg);
}
}
}
void
pmap_bootstrap(
vm_offset_t vstart)
{
pmap_paddr_t pmap_struct_start;
vm_size_t pv_head_size;
vm_size_t pv_lock_table_size;
vm_size_t ptd_root_table_size;
vm_size_t pp_attr_table_size;
vm_size_t io_attr_table_size;
unsigned int niorgns;
unsigned int npages;
unsigned int i;
vm_map_offset_t maxoffset;
#ifdef PMAP_TRACES
if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) {
kprintf("Kernel traces for pmap operations enabled\n");
}
#endif
pmap_stamp = 1;
kernel_pmap->tte = cpu_tte;
kernel_pmap->ttep = cpu_ttep;
#if (__ARM_VMSA__ > 7)
kernel_pmap->min = ARM64_TTBR1_MIN_ADDR;
#else
kernel_pmap->min = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
#endif
kernel_pmap->max = VM_MAX_KERNEL_ADDRESS;
kernel_pmap->wired = 0;
kernel_pmap->ref_count = 1;
kernel_pmap->gc_status = 0;
kernel_pmap->nx_enabled = TRUE;
#ifdef __arm64__
kernel_pmap->is_64bit = TRUE;
#else
kernel_pmap->is_64bit = FALSE;
#endif
kernel_pmap->stamp = hw_atomic_add(&pmap_stamp, 1);
kernel_pmap->nested_region_grand_addr = 0x0ULL;
kernel_pmap->nested_region_subord_addr = 0x0ULL;
kernel_pmap->nested_region_size = 0x0ULL;
kernel_pmap->nested_region_asid_bitmap = NULL;
kernel_pmap->nested_region_asid_bitmap_size = 0x0UL;
#if (__ARM_VMSA__ == 7)
kernel_pmap->tte_index_max = 4*NTTES;
#else
kernel_pmap->tte_index_max = (ARM_PGBYTES / sizeof(tt_entry_t));
#endif
kernel_pmap->prev_tte = (tt_entry_t *) NULL;
kernel_pmap->cpu_ref = 0;
PMAP_LOCK_INIT(kernel_pmap);
#if (__ARM_VMSA__ == 7)
simple_lock_init(&kernel_pmap->tt1_lock, 0);
#endif
memset((void *) &kernel_pmap->stats, 0, sizeof(kernel_pmap->stats));
niorgns = pmap_compute_io_rgns();
npages = (unsigned int)atop(mem_size);
pp_attr_table_size = npages * sizeof(pp_attr_t);
io_attr_table_size = niorgns * sizeof(io_attr_t);
pv_lock_table_size = npages;
pv_head_size = round_page(sizeof(pv_entry_t *) * npages);
#if (__ARM_VMSA__ == 7)
ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+12));
#else
ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+13));
#endif
pmap_struct_start = avail_start;
pp_attr_table = (pp_attr_t *) phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + pp_attr_table_size, __alignof(pp_attr_t));
io_attr_table = (io_attr_t *) phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + io_attr_table_size + pv_lock_table_size, __alignof(pv_entry_t*));
pv_head_table = (pv_entry_t **) phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + pv_head_size, __alignof(pt_desc_t));
ptd_root_table = (pt_desc_t *)phystokv(avail_start);
avail_start = round_page(avail_start + ptd_root_table_size);
memset((char *)phystokv(pmap_struct_start), 0, avail_start - pmap_struct_start);
pmap_load_io_rgns();
ptd_bootstrap(ptd_root_table, (unsigned int)(ptd_root_table_size/sizeof(pt_desc_t)));
pmap_cpu_data_array_init();
vm_first_phys = gPhysBase;
vm_last_phys = trunc_page(avail_end);
simple_lock_init(&pmaps_lock, 0);
queue_init(&map_pmap_list);
queue_enter(&map_pmap_list, kernel_pmap, pmap_t, pmaps);
queue_init(&tt_pmap_list);
tt_pmap_count = 0;
tt_pmap_max = 0;
free_page_size_tt_list = TT_FREE_ENTRY_NULL;
free_page_size_tt_count = 0;
free_page_size_tt_max = 0;
free_two_page_size_tt_list = TT_FREE_ENTRY_NULL;
free_two_page_size_tt_count = 0;
free_two_page_size_tt_max = 0;
free_tt_list = TT_FREE_ENTRY_NULL;
free_tt_count = 0;
free_tt_max = 0;
simple_lock_init(&pt_pages_lock, 0);
queue_init(&pt_page_list);
simple_lock_init(&pmap_pages_lock, 0);
pmap_pages_request_count = 0;
pmap_pages_request_acum = 0;
pmap_pages_reclaim_list = PAGE_FREE_ENTRY_NULL;
virtual_space_start = vstart;
virtual_space_end = VM_MAX_KERNEL_ADDRESS;
for (i = 0; i < MAX_ASID / (sizeof(uint32_t) * NBBY); i++)
asid_bitmap[i] = 0xffffffff;
for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) {
asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY)));
#if __ARM_KERNEL_PROTECT__
assert((i + 1) < MAX_ASID);
asid_bitmap[(i + 1) / (sizeof(uint32_t) * NBBY)] &= ~(1 << ((i + 1) % (sizeof(uint32_t) * NBBY)));
#endif
}
kernel_pmap->asid = 0;
kernel_pmap->vasid = 0;
if (PE_parse_boot_argn("arm_maxoffset", &maxoffset, sizeof (maxoffset))) {
maxoffset = trunc_page(maxoffset);
if ((maxoffset >= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MIN))
&& (maxoffset <= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MAX))) {
arm_pmap_max_offset_default = maxoffset;
}
}
#if defined(__arm64__)
if (PE_parse_boot_argn("arm64_maxoffset", &maxoffset, sizeof (maxoffset))) {
maxoffset = trunc_page(maxoffset);
if ((maxoffset >= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MIN))
&& (maxoffset <= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MAX))) {
arm64_pmap_max_offset_default = maxoffset;
}
}
#endif
#if DEVELOPMENT || DEBUG
PE_parse_boot_argn("panic_on_unsigned_execute", &panic_on_unsigned_execute, sizeof (panic_on_unsigned_execute));
#endif
pmap_nesting_size_min = ARM_NESTING_SIZE_MIN;
pmap_nesting_size_max = ARM_NESTING_SIZE_MAX;
simple_lock_init(&phys_backup_lock, 0);
#if MACH_ASSERT
PE_parse_boot_argn("pmap_stats_assert",
&pmap_stats_assert,
sizeof (pmap_stats_assert));
#endif
}
void
pmap_virtual_space(
vm_offset_t *startp,
vm_offset_t *endp
)
{
*startp = virtual_space_start;
*endp = virtual_space_end;
}
boolean_t
pmap_virtual_region(
unsigned int region_select,
vm_map_offset_t *startp,
vm_map_size_t *size
)
{
boolean_t ret = FALSE;
#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__
if (region_select == 0) {
#if (__ARM_VMSA__ > 7)
*startp = LOW_GLOBAL_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK;
#else
#error Unsupported configuration
#endif
*size = ((VM_MAX_KERNEL_ADDRESS - *startp) & ~PAGE_MASK);
ret = TRUE;
}
#else
#if (__ARM_VMSA__ > 7)
unsigned long low_global_vr_mask = 0;
vm_map_size_t low_global_vr_size = 0;
#endif
if (region_select == 0) {
#if (__ARM_VMSA__ == 7)
*startp = gVirtBase & 0xFFC00000;
*size = ((virtual_space_start-(gVirtBase & 0xFFC00000)) + ~0xFFC00000) & 0xFFC00000;
#else
if (!TEST_PAGE_SIZE_4K) {
*startp = gVirtBase & 0xFFFFFFFFFE000000;
*size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFE000000)) + ~0xFFFFFFFFFE000000) & 0xFFFFFFFFFE000000;
} else {
*startp = gVirtBase & 0xFFFFFFFFFF800000;
*size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFF800000)) + ~0xFFFFFFFFFF800000) & 0xFFFFFFFFFF800000;
}
#endif
ret = TRUE;
}
if (region_select == 1) {
*startp = VREGION1_START;
*size = VREGION1_SIZE;
ret = TRUE;
}
#if (__ARM_VMSA__ > 7)
if (!TEST_PAGE_SIZE_4K) {
low_global_vr_mask = 0xFFFFFFFFFE000000;
low_global_vr_size = 0x2000000;
} else {
low_global_vr_mask = 0xFFFFFFFFFF800000;
low_global_vr_size = 0x800000;
}
if (((gVirtBase & low_global_vr_mask) != LOW_GLOBAL_BASE_ADDRESS) && (region_select == 2)) {
*startp = LOW_GLOBAL_BASE_ADDRESS;
*size = low_global_vr_size;
ret = TRUE;
}
if (region_select == 3) {
*startp = VM_MIN_KERNEL_ADDRESS;
*size = LOW_GLOBAL_BASE_ADDRESS - *startp;
ret = TRUE;
}
#endif
#endif
return ret;
}
unsigned int
pmap_free_pages(
void)
{
return (unsigned int)atop(avail_end - first_avail);
}
boolean_t
pmap_next_page_hi(
ppnum_t * pnum)
{
return pmap_next_page(pnum);
}
boolean_t
pmap_next_page(
ppnum_t *pnum)
{
if (first_avail != avail_end) {
*pnum = (ppnum_t)atop(first_avail);
first_avail += PAGE_SIZE;
return TRUE;
}
return FALSE;
}
void
pmap_init(
void)
{
vm_protect(kernel_map, 0, PAGE_SIZE, TRUE, VM_PROT_NONE);
pmap_initialized = TRUE;
pmap_zone_init();
_vm_object_allocate(mem_size, pmap_object);
pmap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
pv_init();
assert(hard_maxproc < MAX_ASID);
#if CONFIG_PGTRACE
pmap_pgtrace_init();
#endif
}
boolean_t
pmap_verify_free(
ppnum_t ppnum)
{
pv_entry_t **pv_h;
int pai;
boolean_t result = TRUE;
pmap_paddr_t phys = ptoa(ppnum);
assert(phys != vm_page_fictitious_addr);
if (!pa_valid(phys))
return (FALSE);
pai = (int)pa_index(phys);
pv_h = pai_to_pvh(pai);
result = (pvh_list(pv_h) == PV_ENTRY_NULL);
return (result);
}
static void
pmap_zone_init(
void)
{
pmap_zone = zinit((vm_size_t) sizeof(struct pmap), (vm_size_t) sizeof(struct pmap)*256,
PAGE_SIZE, "pmap");
}
static pmap_t
pmap_create_internal(
ledger_t ledger,
vm_map_size_t size,
boolean_t is_64bit)
{
unsigned i;
pmap_t p;
if (size != 0) {
return (PMAP_NULL);
}
if ((p = (pmap_t) zalloc(pmap_zone)) == PMAP_NULL)
return (PMAP_NULL);
if (is_64bit) {
p->min = MACH_VM_MIN_ADDRESS;
p->max = MACH_VM_MAX_ADDRESS;
} else {
p->min = VM_MIN_ADDRESS;
p->max = VM_MAX_ADDRESS;
}
p->wired = 0;
p->ref_count = 1;
p->gc_status = 0;
p->stamp = hw_atomic_add(&pmap_stamp, 1);
p->nx_enabled = TRUE;
p->is_64bit = is_64bit;
p->nested = FALSE;
p->nested_pmap = PMAP_NULL;
ledger_reference(ledger);
p->ledger = ledger;
PMAP_LOCK_INIT(p);
#if (__ARM_VMSA__ == 7)
simple_lock_init(&p->tt1_lock, 0);
#endif
memset((void *) &p->stats, 0, sizeof(p->stats));
p->tt_entry_free = (tt_entry_t *)0;
p->tte = pmap_tt1_allocate(p, PMAP_ROOT_ALLOC_SIZE, 0);
p->ttep = ml_static_vtop((vm_offset_t)p->tte);
#if (__ARM_VMSA__ == 7)
p->tte_index_max = NTTES;
#else
p->tte_index_max = (PMAP_ROOT_ALLOC_SIZE / sizeof(tt_entry_t));
#endif
p->prev_tte = (tt_entry_t *) NULL;
p->cpu_ref = 0;
for (i = 0; i < p->tte_index_max; i++)
p->tte[i] = ARM_TTE_TYPE_FAULT;
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) (p->tte), PMAP_ROOT_ALLOC_SIZE);
#else
__asm__ volatile("dsb ish");
#endif
p->vasid = alloc_asid();
p->asid = p->vasid % ARM_MAX_ASID;
p->nested_region_grand_addr = 0x0ULL;
p->nested_region_subord_addr = 0x0ULL;
p->nested_region_size = 0x0ULL;
p->nested_region_asid_bitmap = NULL;
p->nested_region_asid_bitmap_size = 0x0UL;
#if MACH_ASSERT
p->pmap_stats_assert = TRUE;
p->pmap_pid = 0;
strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
#endif
#if DEVELOPMENT || DEBUG
p->footprint_suspended = FALSE;
p->footprint_was_suspended = FALSE;
#endif
simple_lock(&pmaps_lock);
queue_enter(&map_pmap_list, p, pmap_t, pmaps);
simple_unlock(&pmaps_lock);
return (p);
}
pmap_t
pmap_create(
ledger_t ledger,
vm_map_size_t size,
boolean_t is_64bit)
{
pmap_t pmap;
PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, size, is_64bit);
pmap = pmap_create_internal(ledger, size, is_64bit);
PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END,
VM_KERNEL_ADDRHIDE(pmap));
return pmap;
}
#if MACH_ASSERT
static void
pmap_set_process_internal(
__unused pmap_t pmap,
__unused int pid,
__unused char *procname)
{
#if MACH_ASSERT
if (pmap == NULL) {
return;
}
pmap->pmap_pid = pid;
strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
if (!strncmp(procname, "corecaptured", sizeof (pmap->pmap_procname))) {
pmap->pmap_stats_assert = FALSE;
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.phys_footprint);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.internal);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.internal_compressed);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.iokit_mapped);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.alternate_accounting);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.alternate_accounting_compressed);
}
#endif
}
#endif
#if MACH_ASSERT
void
pmap_set_process(
pmap_t pmap,
int pid,
char *procname)
{
pmap_set_process_internal(pmap, pid, procname);
}
struct {
uint64_t num_pmaps_checked;
int phys_footprint_over;
ledger_amount_t phys_footprint_over_total;
ledger_amount_t phys_footprint_over_max;
int phys_footprint_under;
ledger_amount_t phys_footprint_under_total;
ledger_amount_t phys_footprint_under_max;
int internal_over;
ledger_amount_t internal_over_total;
ledger_amount_t internal_over_max;
int internal_under;
ledger_amount_t internal_under_total;
ledger_amount_t internal_under_max;
int internal_compressed_over;
ledger_amount_t internal_compressed_over_total;
ledger_amount_t internal_compressed_over_max;
int internal_compressed_under;
ledger_amount_t internal_compressed_under_total;
ledger_amount_t internal_compressed_under_max;
int iokit_mapped_over;
ledger_amount_t iokit_mapped_over_total;
ledger_amount_t iokit_mapped_over_max;
int iokit_mapped_under;
ledger_amount_t iokit_mapped_under_total;
ledger_amount_t iokit_mapped_under_max;
int alternate_accounting_over;
ledger_amount_t alternate_accounting_over_total;
ledger_amount_t alternate_accounting_over_max;
int alternate_accounting_under;
ledger_amount_t alternate_accounting_under_total;
ledger_amount_t alternate_accounting_under_max;
int alternate_accounting_compressed_over;
ledger_amount_t alternate_accounting_compressed_over_total;
ledger_amount_t alternate_accounting_compressed_over_max;
int alternate_accounting_compressed_under;
ledger_amount_t alternate_accounting_compressed_under_total;
ledger_amount_t alternate_accounting_compressed_under_max;
int page_table_over;
ledger_amount_t page_table_over_total;
ledger_amount_t page_table_over_max;
int page_table_under;
ledger_amount_t page_table_under_total;
ledger_amount_t page_table_under_max;
int purgeable_volatile_over;
ledger_amount_t purgeable_volatile_over_total;
ledger_amount_t purgeable_volatile_over_max;
int purgeable_volatile_under;
ledger_amount_t purgeable_volatile_under_total;
ledger_amount_t purgeable_volatile_under_max;
int purgeable_nonvolatile_over;
ledger_amount_t purgeable_nonvolatile_over_total;
ledger_amount_t purgeable_nonvolatile_over_max;
int purgeable_nonvolatile_under;
ledger_amount_t purgeable_nonvolatile_under_total;
ledger_amount_t purgeable_nonvolatile_under_max;
int purgeable_volatile_compressed_over;
ledger_amount_t purgeable_volatile_compressed_over_total;
ledger_amount_t purgeable_volatile_compressed_over_max;
int purgeable_volatile_compressed_under;
ledger_amount_t purgeable_volatile_compressed_under_total;
ledger_amount_t purgeable_volatile_compressed_under_max;
int purgeable_nonvolatile_compressed_over;
ledger_amount_t purgeable_nonvolatile_compressed_over_total;
ledger_amount_t purgeable_nonvolatile_compressed_over_max;
int purgeable_nonvolatile_compressed_under;
ledger_amount_t purgeable_nonvolatile_compressed_under_total;
ledger_amount_t purgeable_nonvolatile_compressed_under_max;
} pmap_ledgers_drift;
#endif
static void
pmap_destroy_internal(
pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
pt_entry_t *ttep;
unsigned int i;
pmap_t tmp_pmap, tt_pmap;
queue_head_t tmp_pmap_list;
queue_init(&tmp_pmap_list);
simple_lock(&pmaps_lock);
tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tt_pmap_list));
while (!queue_end(&tt_pmap_list, (queue_entry_t)tt_pmap)) {
if (tt_pmap->cpu_ref == 0 ) {
tmp_pmap = tt_pmap;
tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps));
queue_remove(&tt_pmap_list, tmp_pmap, pmap_t, pmaps);
tt_pmap_count--;
queue_enter(&tmp_pmap_list, tmp_pmap, pmap_t, pmaps);
} else {
tmp_pmap = tt_pmap;
tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps));
}
}
simple_unlock(&pmaps_lock);
tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tmp_pmap_list));
while (!queue_end(&tmp_pmap_list, (queue_entry_t)tmp_pmap)) {
tt_pmap = tmp_pmap;
tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tt_pmap->pmaps));
queue_remove(&tmp_pmap_list, tt_pmap, pmap_t, pmaps);
if (tt_pmap->tte) {
pmap_tt1_deallocate(pmap, tt_pmap->tte, tt_pmap->tte_index_max*sizeof(tt_entry_t), 0);
tt_pmap->tte = (tt_entry_t *) NULL;
tt_pmap->ttep = 0;
tt_pmap->tte_index_max = 0;
}
if (tt_pmap->prev_tte) {
pmap_tt1_deallocate(pmap, tt_pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0);
tt_pmap->prev_tte = (tt_entry_t *) NULL;
}
assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
free_asid(tt_pmap->vasid);
pmap_check_ledgers(tt_pmap);
ledger_dereference(tt_pmap->ledger);
zfree(pmap_zone, tt_pmap);
}
if (pmap == PMAP_NULL)
return;
if (hw_atomic_sub(&pmap->ref_count, 1) != 0)
return;
simple_lock(&pmaps_lock);
while (pmap->gc_status & PMAP_GC_INFLIGHT) {
pmap->gc_status |= PMAP_GC_WAIT;
assert_wait((event_t) & pmap->gc_status, THREAD_UNINT);
simple_unlock(&pmaps_lock);
(void) thread_block(THREAD_CONTINUE_NULL);
simple_lock(&pmaps_lock);
}
queue_remove(&map_pmap_list, pmap, pmap_t, pmaps);
simple_unlock(&pmaps_lock);
PMAP_LOCK(pmap);
for (i = 0; i < pmap->tte_index_max; i++) {
ttep = &pmap->tte[i];
if ((*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL);
flush_mmu_tlb_entry((i<<ARM_TT_L1_SHIFT) | (pmap->asid & 0xff));
}
}
PMAP_UNLOCK(pmap);
if (pmap->cpu_ref == 0) {
if (pmap->tte) {
pmap_tt1_deallocate(pmap, pmap->tte, pmap->tte_index_max*sizeof(tt_entry_t), 0);
pmap->tte = (tt_entry_t *) NULL;
pmap->ttep = 0;
pmap->tte_index_max = 0;
}
if (pmap->prev_tte) {
pmap_tt1_deallocate(pmap, pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0);
pmap->prev_tte = (tt_entry_t *) NULL;
}
assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
free_asid(pmap->vasid);
pmap_check_ledgers(pmap);
ledger_dereference(pmap->ledger);
if (pmap->nested_region_asid_bitmap)
kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int));
zfree(pmap_zone, pmap);
} else {
simple_lock(&pmaps_lock);
queue_enter(&tt_pmap_list, pmap, pmap_t, pmaps);
tt_pmap_count++;
if (tt_pmap_count > tt_pmap_max)
tt_pmap_max = tt_pmap_count;
simple_unlock(&pmaps_lock);
}
#else
pt_entry_t *ttep;
pmap_paddr_t pa;
vm_map_address_t c;
if (pmap == PMAP_NULL) {
return;
}
pmap_unmap_sharedpage(pmap);
if (hw_atomic_sub(&pmap->ref_count, 1) == 0) {
simple_lock(&pmaps_lock);
while (pmap->gc_status & PMAP_GC_INFLIGHT) {
pmap->gc_status |= PMAP_GC_WAIT;
assert_wait((event_t) & pmap->gc_status, THREAD_UNINT);
simple_unlock(&pmaps_lock);
(void) thread_block(THREAD_CONTINUE_NULL);
simple_lock(&pmaps_lock);
}
queue_remove(&map_pmap_list, pmap, pmap_t, pmaps);
simple_unlock(&pmaps_lock);
for (c = pmap->min; c < pmap->max; c += ARM_TT_L2_SIZE) {
ttep = pmap_tt2e(pmap, c);
if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
PMAP_LOCK(pmap);
pmap_tte_deallocate(pmap, ttep, PMAP_TT_L2_LEVEL);
PMAP_UNLOCK(pmap);
flush_mmu_tlb_entry(tlbi_addr(c) | tlbi_asid(pmap->asid));
}
}
#if !__ARM64_TWO_LEVEL_PMAP__
for (c = pmap->min; c < pmap->max; c += ARM_TT_L1_SIZE) {
ttep = pmap_tt1e(pmap, c);
if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
PMAP_LOCK(pmap);
pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL);
PMAP_UNLOCK(pmap);
}
}
#endif
if (pmap->tte) {
pa = pmap->ttep;
pmap_tt1_deallocate(pmap, (tt_entry_t *)phystokv(pa), PMAP_ROOT_ALLOC_SIZE, 0);
}
assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT);
free_asid(pmap->vasid);
if (pmap->nested_region_asid_bitmap) {
kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int));
}
pmap_check_ledgers(pmap);
ledger_dereference(pmap->ledger);
zfree(pmap_zone, pmap);
}
#endif
}
void
pmap_destroy(
pmap_t pmap)
{
PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap));
pmap_destroy_internal(pmap);
PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
}
static void
pmap_reference_internal(
pmap_t pmap)
{
if (pmap != PMAP_NULL) {
(void) hw_atomic_add(&pmap->ref_count, 1);
}
}
void
pmap_reference(
pmap_t pmap)
{
pmap_reference_internal(pmap);
}
static tt_entry_t *
pmap_tt1_allocate(
pmap_t pmap,
vm_size_t size,
unsigned option)
{
tt_entry_t *tt1;
tt_free_entry_t *tt1_free;
pmap_paddr_t pa;
vm_address_t va;
vm_address_t va_end;
kern_return_t ret;
simple_lock(&pmaps_lock);
if ((size == PAGE_SIZE) && (free_page_size_tt_count != 0)) {
free_page_size_tt_count--;
tt1 = (tt_entry_t *)free_page_size_tt_list;
free_page_size_tt_list = ((tt_free_entry_t *)tt1)->next;
simple_unlock(&pmaps_lock);
pmap_tt_ledger_credit(pmap, size);
return (tt_entry_t *)tt1;
};
if ((size == 2*PAGE_SIZE) && (free_two_page_size_tt_count != 0)) {
free_two_page_size_tt_count--;
tt1 = (tt_entry_t *)free_two_page_size_tt_list;
free_two_page_size_tt_list = ((tt_free_entry_t *)tt1)->next;
simple_unlock(&pmaps_lock);
pmap_tt_ledger_credit(pmap, size);
return (tt_entry_t *)tt1;
};
if (free_tt_count != 0) {
free_tt_count--;
tt1 = (tt_entry_t *)free_tt_list;
free_tt_list = (tt_free_entry_t *)((tt_free_entry_t *)tt1)->next;
simple_unlock(&pmaps_lock);
pmap_tt_ledger_credit(pmap, size);
return (tt_entry_t *)tt1;
}
simple_unlock(&pmaps_lock);
ret = pmap_pages_alloc(&pa, (unsigned)((size < PAGE_SIZE)? PAGE_SIZE : size), ((option & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0));
if(ret == KERN_RESOURCE_SHORTAGE)
return (tt_entry_t *)0;
if (size < PAGE_SIZE) {
simple_lock(&pmaps_lock);
for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + size; va < va_end; va = va+size) {
tt1_free = (tt_free_entry_t *)va;
tt1_free->next = free_tt_list;
free_tt_list = tt1_free;
free_tt_count++;
}
if (free_tt_count > free_tt_max)
free_tt_max = free_tt_count;
simple_unlock(&pmaps_lock);
}
OSAddAtomic((uint32_t)(size / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
OSAddAtomic64(size / PMAP_ROOT_ALLOC_SIZE, &alloc_tteroot_count);
pmap_tt_ledger_credit(pmap, size);
return (tt_entry_t *) phystokv(pa);
}
static void
pmap_tt1_deallocate(
pmap_t pmap,
tt_entry_t *tt,
vm_size_t size,
unsigned option)
{
tt_free_entry_t *tt_entry;
tt_entry = (tt_free_entry_t *)tt;
if (not_in_kdp)
simple_lock(&pmaps_lock);
if (size < PAGE_SIZE) {
free_tt_count++;
if (free_tt_count > free_tt_max)
free_tt_max = free_tt_count;
tt_entry->next = free_tt_list;
free_tt_list = tt_entry;
}
if (size == PAGE_SIZE) {
free_page_size_tt_count++;
if (free_page_size_tt_count > free_page_size_tt_max)
free_page_size_tt_max = free_page_size_tt_count;
tt_entry->next = free_page_size_tt_list;
free_page_size_tt_list = tt_entry;
}
if (size == 2*PAGE_SIZE) {
free_two_page_size_tt_count++;
if (free_two_page_size_tt_count > free_two_page_size_tt_max)
free_two_page_size_tt_max = free_two_page_size_tt_count;
tt_entry->next = free_two_page_size_tt_list;
free_two_page_size_tt_list = tt_entry;
}
if ((option & PMAP_TT_DEALLOCATE_NOBLOCK) || (!not_in_kdp)) {
if (not_in_kdp)
simple_unlock(&pmaps_lock);
pmap_tt_ledger_debit(pmap, size);
return;
}
while (free_page_size_tt_count > FREE_PAGE_SIZE_TT_MAX) {
free_page_size_tt_count--;
tt = (tt_entry_t *)free_page_size_tt_list;
free_page_size_tt_list = ((tt_free_entry_t *)tt)->next;
simple_unlock(&pmaps_lock);
pmap_pages_free(ml_static_vtop((vm_offset_t)tt), PAGE_SIZE);
OSAddAtomic(-(int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
simple_lock(&pmaps_lock);
}
while (free_two_page_size_tt_count > FREE_TWO_PAGE_SIZE_TT_MAX) {
free_two_page_size_tt_count--;
tt = (tt_entry_t *)free_two_page_size_tt_list;
free_two_page_size_tt_list = ((tt_free_entry_t *)tt)->next;
simple_unlock(&pmaps_lock);
pmap_pages_free(ml_static_vtop((vm_offset_t)tt), 2*PAGE_SIZE);
OSAddAtomic(-2 * (int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
simple_lock(&pmaps_lock);
}
simple_unlock(&pmaps_lock);
pmap_tt_ledger_debit(pmap, size);
}
static kern_return_t
pmap_tt_allocate(
pmap_t pmap,
tt_entry_t **ttp,
unsigned int level,
unsigned int options)
{
pmap_paddr_t pa;
*ttp = NULL;
PMAP_LOCK(pmap);
if ((tt_free_entry_t *)pmap->tt_entry_free != NULL) {
tt_free_entry_t *tt_free_next;
tt_free_next = ((tt_free_entry_t *)pmap->tt_entry_free)->next;
*ttp = (tt_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = (tt_entry_t *)tt_free_next;
}
PMAP_UNLOCK(pmap);
if (*ttp == NULL) {
pt_desc_t *ptdp;
while (pmap_pages_alloc(&pa, PAGE_SIZE, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if(options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
if (level < PMAP_TT_MAX_LEVEL) {
OSAddAtomic64(1, &alloc_ttepages_count);
OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count));
} else {
OSAddAtomic64(1, &alloc_ptepages_count);
OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count));
}
pmap_tt_ledger_credit(pmap, PAGE_SIZE);
PMAP_ZINFO_PALLOC(pmap, PAGE_SIZE);
ptdp = ptd_alloc(pmap);
*(pt_desc_t **)pai_to_pvh(pa_index(pa)) = ptdp;
__unreachable_ok_push
if (TEST_PAGE_RATIO_4) {
vm_address_t va;
vm_address_t va_end;
PMAP_LOCK(pmap);
for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + ARM_PGBYTES; va < va_end; va = va+ARM_PGBYTES) {
((tt_free_entry_t *)va)->next = (tt_free_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = (tt_entry_t *)va;
}
PMAP_UNLOCK(pmap);
}
__unreachable_ok_pop
*ttp = (tt_entry_t *)phystokv(pa);
}
return KERN_SUCCESS;
}
static void
pmap_tt_deallocate(
pmap_t pmap,
tt_entry_t *ttp,
unsigned int level)
{
pt_desc_t *ptdp;
unsigned pt_acc_cnt;
unsigned i, max_pt_index = PAGE_RATIO;
vm_offset_t free_page=0;
PMAP_LOCK(pmap);
ptdp = ptep_get_ptd((vm_offset_t)ttp);
if (level < PMAP_TT_MAX_LEVEL) {
if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt == PT_DESC_REFCOUNT)
ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt = 0;
}
ptdp->pt_map[ARM_PT_DESC_INDEX(ttp)].va = 0;
if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt != 0)
panic("pmap_tt_deallocate(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt);
for (i = 0, pt_acc_cnt = 0 ; i < max_pt_index ; i++)
pt_acc_cnt += ptdp->pt_cnt[i].refcnt;
if (pt_acc_cnt == 0) {
tt_free_entry_t *tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free;
unsigned pt_free_entry_cnt = 1;
while (pt_free_entry_cnt < max_pt_index && tt_free_list) {
tt_free_entry_t *tt_free_list_next;
tt_free_list_next = tt_free_list->next;
if ((((vm_offset_t)tt_free_list_next) - ((vm_offset_t)ttp & ~PAGE_MASK)) < PAGE_SIZE) {
pt_free_entry_cnt++;
}
tt_free_list = tt_free_list_next;
}
if (pt_free_entry_cnt == max_pt_index) {
tt_free_entry_t *tt_free_list_cur;
free_page = (vm_offset_t)ttp & ~PAGE_MASK;
tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free;
tt_free_list_cur = (tt_free_entry_t *)&pmap->tt_entry_free;
while (tt_free_list_cur) {
tt_free_entry_t *tt_free_list_next;
tt_free_list_next = tt_free_list_cur->next;
if ((((vm_offset_t)tt_free_list_next) - free_page) < PAGE_SIZE) {
tt_free_list->next = tt_free_list_next->next;
} else {
tt_free_list = tt_free_list_next;
}
tt_free_list_cur = tt_free_list_next;
}
} else {
((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = ttp;
}
} else {
((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = ttp;
}
PMAP_UNLOCK(pmap);
if (free_page != 0) {
ptd_deallocate(ptep_get_ptd((vm_offset_t)free_page));
*(pt_desc_t **)pai_to_pvh(pa_index(ml_static_vtop(free_page))) = NULL;
pmap_pages_free(ml_static_vtop(free_page), PAGE_SIZE);
if (level < PMAP_TT_MAX_LEVEL)
OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count));
else
OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count));
PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
pmap_tt_ledger_debit(pmap, PAGE_SIZE);
}
}
static void
pmap_tte_deallocate(
pmap_t pmap,
tt_entry_t *ttep,
unsigned int level)
{
pmap_paddr_t pa;
tt_entry_t tte;
PMAP_ASSERT_LOCKED(pmap);
tte = *ttep;
if (tte == 0) {
panic("pmap_tte_deallocate(): null tt_entry ttep==%p\n", ttep);
}
#if MACH_ASSERT
if (tte_get_ptd(tte)->pmap != pmap) {
panic("pmap_tte_deallocate(): ptd=%p ptd->pmap=%p pmap=%p \n",
tte_get_ptd(tte), tte_get_ptd(tte)->pmap, pmap);
}
#endif
if (((level+1) == PMAP_TT_MAX_LEVEL) && (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt != 0)) {
panic("pmap_tte_deallocate(): pmap=%p ttep=%p ptd=%p refcnt=0x%x \n", pmap, ttep,
tte_get_ptd(tte), (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt));
}
#if (__ARM_VMSA__ == 7)
{
tt_entry_t *ttep_4M = (tt_entry_t *) ((vm_offset_t)ttep & 0xFFFFFFF0);
unsigned i;
for (i = 0; i<4; i++, ttep_4M++)
*ttep_4M = (tt_entry_t) 0;
}
#else
*ttep = (tt_entry_t) 0;
#endif
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t));
#else
__asm__ volatile("dsb ish");
#endif
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
#if MACH_ASSERT
{
pt_entry_t *pte_p = ((pt_entry_t *) (ttetokv(tte) & ~ARM_PGMASK));
unsigned i;
for (i = 0; i < (ARM_PGBYTES / sizeof(*pte_p)); i++,pte_p++) {
if (ARM_PTE_IS_COMPRESSED(*pte_p)) {
panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx compressed\n",
(uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p));
} else if (((*pte_p) & ARM_PTE_TYPE_MASK) != ARM_PTE_TYPE_FAULT) {
panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx\n",
(uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p));
}
}
}
#endif
PMAP_UNLOCK(pmap);
pa = tte_to_pa(tte) & ~ARM_PGMASK;
pmap_tt_deallocate(pmap, (tt_entry_t *) phystokv(pa), level+1);
PMAP_LOCK(pmap);
}
}
static int
pmap_remove_range(
pmap_t pmap,
vm_map_address_t va,
pt_entry_t *bpte,
pt_entry_t *epte,
uint32_t *rmv_cnt)
{
return pmap_remove_range_options(pmap, va, bpte, epte, rmv_cnt,
PMAP_OPTIONS_REMOVE);
}
#if MACH_ASSERT
int num_reusable_mismatch = 0;
#endif
static int
pmap_remove_range_options(
pmap_t pmap,
vm_map_address_t va,
pt_entry_t *bpte,
pt_entry_t *epte,
uint32_t *rmv_cnt,
int options)
{
pt_entry_t *cpte;
int num_removed, num_unwired;
int num_pte_changed;
int pai = 0;
pmap_paddr_t pa;
int num_external, num_internal, num_reusable;
int num_alt_internal;
uint64_t num_compressed, num_alt_compressed;
PMAP_ASSERT_LOCKED(pmap);
num_removed = 0;
num_unwired = 0;
num_pte_changed = 0;
num_external = 0;
num_internal = 0;
num_reusable = 0;
num_compressed = 0;
num_alt_internal = 0;
num_alt_compressed = 0;
for (cpte = bpte; cpte < epte;
cpte += PAGE_SIZE/ARM_PGBYTES, va += PAGE_SIZE) {
pv_entry_t **pv_h, **pve_pp;
pv_entry_t *pve_p;
pt_entry_t spte;
boolean_t managed=FALSE;
spte = *cpte;
#if CONFIG_PGTRACE
if (pgtrace_enabled) {
pmap_pgtrace_remove_clone(pmap, pte_to_pa(spte), va);
}
#endif
while (!managed) {
if (pmap != kernel_pmap &&
(options & PMAP_OPTIONS_REMOVE) &&
(ARM_PTE_IS_COMPRESSED(spte))) {
num_compressed++;
if (spte & ARM_PTE_COMPRESSED_ALT) {
num_alt_compressed++;
}
WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT);
if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0)
panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte);
spte = *cpte;
}
pa = pte_to_pa(spte);
if (!pa_valid(pa)) {
break;
}
pai = (int)pa_index(pa);
LOCK_PVH(pai);
spte = *cpte;
pa = pte_to_pa(spte);
if (pai == (int)pa_index(pa)) {
managed =TRUE;
break; }
UNLOCK_PVH(pai);
}
if (ARM_PTE_IS_COMPRESSED(*cpte)) {
continue;
}
if (*cpte != ARM_PTE_TYPE_FAULT) {
assert(!ARM_PTE_IS_COMPRESSED(*cpte));
#if MACH_ASSERT
if (managed && (pmap != kernel_pmap) && (ptep_get_va(cpte) != va)) {
panic("pmap_remove_range_options(): cpte=%p ptd=%p pte=0x%llx va=0x%llx\n",
cpte, ptep_get_ptd(cpte), (uint64_t)*cpte, (uint64_t)va);
}
#endif
WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT);
num_pte_changed++;
}
if ((spte != ARM_PTE_TYPE_FAULT) &&
(pmap != kernel_pmap)) {
assert(!ARM_PTE_IS_COMPRESSED(spte));
if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0)
panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte);
if(rmv_cnt) (*rmv_cnt)++;
}
if (pte_is_wired(spte)) {
pte_set_wired(cpte, 0);
num_unwired++;
}
if (!managed)
continue;
ASSERT_PVH_LOCKED(pai); pv_h = pai_to_pvh(pai);
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
if (__builtin_expect((cpte != pvh_ptep(pv_h)), 0))
panic("pmap_remove_range(): cpte=%p (0x%llx) does not match pv_h=%p (%p)\n", cpte, (uint64_t)spte, pv_h, pvh_ptep(pv_h));
if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) {
assert(IS_INTERNAL_PAGE(pai));
num_internal++;
num_alt_internal++;
CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
} else if (IS_INTERNAL_PAGE(pai)) {
if (IS_REUSABLE_PAGE(pai)) {
num_reusable++;
} else {
num_internal++;
}
} else {
num_external++;
}
pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_pp = pv_h;
pve_p = pvh_list(pv_h);
while (pve_p != PV_ENTRY_NULL &&
(pve_get_ptep(pve_p) != cpte)) {
pve_pp = pve_link_field(pve_p);
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
if (__builtin_expect((pve_p == PV_ENTRY_NULL), 0)) {
UNLOCK_PVH(pai);
panic("pmap_remove_range(): cpte=%p (0x%llx) not in pv_h=%p\n", cpte, (uint64_t)spte, pv_h);
}
#if MACH_ASSERT
if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) {
pv_entry_t *check_pve_p = PVE_NEXT_PTR(pve_next(pve_p));
while (check_pve_p != PV_ENTRY_NULL) {
if (pve_get_ptep(check_pve_p) == cpte) {
panic("pmap_remove_range(): duplicate pve entry cpte=%p pmap=%p, pv_h=%p, pve_p=%p, pte=0x%llx, va=0x%llx\n",
cpte, pmap, pv_h, pve_p, (uint64_t)spte, (uint64_t)va);
}
check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p));
}
}
#endif
if (IS_ALTACCT_PAGE(pai, pve_p)) {
assert(IS_INTERNAL_PAGE(pai));
num_internal++;
num_alt_internal++;
CLR_ALTACCT_PAGE(pai, pve_p);
} else if (IS_INTERNAL_PAGE(pai)) {
if (IS_REUSABLE_PAGE(pai)) {
num_reusable++;
} else {
num_internal++;
}
} else {
num_external++;
}
pvh_remove(pv_h, pve_pp, pve_p) ;
pv_free(pve_p);
} else {
panic("pmap_remove_range(): unexpected PV head %p, cpte=%p pmap=%p pv_h=%p pte=0x%llx va=0x%llx\n",
*pv_h, cpte, pmap, pv_h, (uint64_t)spte, (uint64_t)va);
}
UNLOCK_PVH(pai);
num_removed++;
}
OSAddAtomic(-num_removed, (SInt32 *) &pmap->stats.resident_count);
pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed));
if (pmap != kernel_pmap) {
#if MACH_ASSERT
if (pmap->stats.internal < num_internal) {
if ((! pmap_stats_assert) ||
(pmap->stats.internal + pmap->stats.reusable) ==
(num_internal + num_reusable)) {
num_reusable_mismatch++;
printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n",
pmap,
(uint64_t) va,
bpte,
epte,
options,
num_internal,
num_removed,
num_unwired,
num_external,
num_reusable,
num_compressed,
num_alt_internal,
num_alt_compressed,
num_pte_changed,
pmap->stats.internal,
pmap->stats.reusable);
num_internal = pmap->stats.internal;
num_reusable = pmap->stats.reusable;
} else {
panic("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d",
pmap,
(uint64_t) va,
bpte,
epte,
options,
num_internal,
num_removed,
num_unwired,
num_external,
num_reusable,
num_compressed,
num_alt_internal,
num_alt_compressed,
num_pte_changed,
pmap->stats.internal,
pmap->stats.reusable);
}
}
#endif
PMAP_STATS_ASSERTF(pmap->stats.external >= num_external,
pmap,
"pmap=%p num_external=%d stats.external=%d",
pmap, num_external, pmap->stats.external);
PMAP_STATS_ASSERTF(pmap->stats.internal >= num_internal,
pmap,
"pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
pmap,
num_internal, pmap->stats.internal,
num_reusable, pmap->stats.reusable);
PMAP_STATS_ASSERTF(pmap->stats.reusable >= num_reusable,
pmap,
"pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
pmap,
num_internal, pmap->stats.internal,
num_reusable, pmap->stats.reusable);
PMAP_STATS_ASSERTF(pmap->stats.compressed >= num_compressed,
pmap,
"pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld",
pmap, num_compressed, num_alt_compressed,
pmap->stats.compressed);
OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count);
if (num_external)
OSAddAtomic(-num_external, &pmap->stats.external);
if (num_internal)
OSAddAtomic(-num_internal, &pmap->stats.internal);
if (num_reusable)
OSAddAtomic(-num_reusable, &pmap->stats.reusable);
if (num_compressed)
OSAddAtomic64(-num_compressed, &pmap->stats.compressed);
pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired));
pmap_ledger_debit(pmap, task_ledgers.internal, machine_ptob(num_internal));
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, machine_ptob(num_alt_internal));
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting_compressed, machine_ptob(num_alt_compressed));
pmap_ledger_debit(pmap, task_ledgers.internal_compressed, machine_ptob(num_compressed));
pmap_ledger_debit(pmap, task_ledgers.phys_footprint,
machine_ptob((num_internal -
num_alt_internal) +
(num_compressed -
num_alt_compressed)));
}
if (num_pte_changed > 0)
FLUSH_PTE_RANGE(bpte, epte);
return num_pte_changed;
}
void
pmap_remove(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end)
{
pmap_remove_options(pmap, start, end, PMAP_OPTIONS_REMOVE);
}
static int
pmap_remove_options_internal(pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
int options)
{
int remove_count = 0;
pt_entry_t *bpte, *epte;
pt_entry_t *pte_p;
tt_entry_t *tte_p;
uint32_t rmv_spte=0;
PMAP_LOCK(pmap);
tte_p = pmap_tte(pmap, start);
if (tte_p == (tt_entry_t *) NULL) {
goto done;
}
if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[ptenum(start)];
epte = bpte + ((end - start) >> ARM_TT_LEAF_SHIFT);
remove_count += pmap_remove_range_options(pmap, start, bpte, epte,
&rmv_spte, options);
#if (__ARM_VMSA__ == 7)
if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) &&
(pmap != kernel_pmap) && (pmap->nested == FALSE)) {
pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL);
flush_mmu_tlb_entry((start & ~ARM_TT_L1_OFFMASK) | (pmap->asid & 0xff));
}
#else
if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) &&
(pmap != kernel_pmap) && (pmap->nested == FALSE)) {
pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL);
flush_mmu_tlb_entry(tlbi_addr(start & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
}
#endif
}
done:
PMAP_UNLOCK(pmap);
return remove_count;
}
void
pmap_remove_options(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
int options)
{
int remove_count = 0;
vm_map_address_t va;
if (pmap == PMAP_NULL)
return;
PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start),
VM_KERNEL_ADDRHIDE(end));
#if MACH_ASSERT
if ((start|end) & PAGE_MASK) {
panic("pmap_remove_options() pmap %p start 0x%llx end 0x%llx\n",
pmap, (uint64_t)start, (uint64_t)end);
}
if ((end < start) || (start < pmap->min) || (end > pmap->max)) {
panic("pmap_remove_options(): invalid address range, pmap=%p, start=0x%llx, end=0x%llx\n",
pmap, (uint64_t)start, (uint64_t)end);
}
#endif
va = start;
while (va < end) {
vm_map_address_t l;
#if (__ARM_VMSA__ == 7)
l = ((va + ARM_TT_L1_SIZE) & ~ARM_TT_L1_OFFMASK);
#else
l = ((va + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK);
#endif
if (l > end)
l = end;
remove_count += pmap_remove_options_internal(pmap, va, l, options);
va = l;
}
if (remove_count > 0)
PMAP_UPDATE_TLBS(pmap, start, end);
PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END);
}
void
pmap_remove_some_phys(
__unused pmap_t map,
__unused ppnum_t pn)
{
}
void
pmap_set_pmap(
pmap_t pmap,
#if !__ARM_USER_PROTECT__
__unused
#endif
thread_t thread)
{
pmap_switch(pmap);
#if __ARM_USER_PROTECT__
if (pmap->tte_index_max == NTTES) {
thread->machine.uptw_ttc = 2;
thread->machine.uptw_ttb = ((unsigned int) pmap->ttep) | TTBR_SETUP;
} else {
thread->machine.uptw_ttc = 1; \
thread->machine.uptw_ttb = ((unsigned int) pmap->ttep ) | TTBR_SETUP;
}
thread->machine.asid = pmap->asid;
#endif
}
static void
pmap_flush_core_tlb_asid(pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
flush_core_tlb_asid(pmap->asid);
#else
flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT);
#if __ARM_KERNEL_PROTECT__
flush_core_tlb_asid(((uint64_t) pmap->asid + 1) << TLBI_ASID_SHIFT);
#endif
#endif
}
static void
pmap_switch_internal(
pmap_t pmap)
{
pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data();
uint32_t last_asid_high_bits, asid_high_bits;
pmap_t cur_pmap;
pmap_t cur_user_pmap;
boolean_t do_asid_flush = FALSE;
#if (__ARM_VMSA__ == 7)
if (not_in_kdp)
simple_lock(&pmap->tt1_lock);
#endif
cur_pmap = current_pmap();
cur_user_pmap = cpu_data_ptr->cpu_user_pmap;
assert(pmap->asid < (sizeof(cpu_data_ptr->cpu_asid_high_bits) / sizeof(*cpu_data_ptr->cpu_asid_high_bits)));
asid_high_bits = pmap->vasid >> ARM_ASID_SHIFT;
last_asid_high_bits = (uint32_t) cpu_data_ptr->cpu_asid_high_bits[pmap->asid];
if (asid_high_bits != last_asid_high_bits) {
cpu_data_ptr->cpu_asid_high_bits[pmap->asid] = (uint8_t) asid_high_bits;
do_asid_flush = TRUE;
}
if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) {
if (cpu_data_ptr->cpu_user_pmap_stamp == pmap->stamp) {
pmap_switch_user_ttb_internal(pmap);
#if (__ARM_VMSA__ == 7)
if (not_in_kdp)
simple_unlock(&pmap->tt1_lock);
#endif
if (do_asid_flush) {
pmap_flush_core_tlb_asid(pmap);
}
return;
} else
cur_user_pmap = NULL;
} else if ((cur_user_pmap == pmap) && (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp))
cur_user_pmap = NULL;
pmap_switch_user_ttb_internal(pmap);
if (do_asid_flush) {
pmap_flush_core_tlb_asid(pmap);
}
#if (__ARM_VMSA__ == 7)
if (not_in_kdp)
simple_unlock(&pmap->tt1_lock);
#else
if (pmap != kernel_pmap) {
if (cur_user_pmap != PMAP_NULL) {
if (pmap_is_64bit(pmap) && !pmap_is_64bit(cur_user_pmap)) {
pmap_sharedpage_flush_32_to_64();
}
} else
flush_core_tlb();
}
#endif
}
void
pmap_switch(
pmap_t pmap)
{
pmap_switch_internal(pmap);
}
void
pmap_page_protect(
ppnum_t ppnum,
vm_prot_t prot)
{
pmap_page_protect_options(ppnum, prot, 0, NULL);
}
static void
pmap_page_protect_options_internal(
ppnum_t ppnum,
vm_prot_t prot,
unsigned int options)
{
pmap_paddr_t phys = ptoa(ppnum);
pv_entry_t **pv_h;
pv_entry_t *pve_p;
pv_entry_t *pveh_p;
pv_entry_t *pvet_p;
pt_entry_t *pte_p;
int pai;
boolean_t remove;
boolean_t set_NX;
unsigned int pvh_cnt = 0;
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return;
}
switch (prot) {
case VM_PROT_ALL:
return;
case VM_PROT_READ:
case VM_PROT_READ | VM_PROT_EXECUTE:
remove = FALSE;
break;
default:
remove = TRUE;
break;
}
pai = (int)pa_index(phys);
LOCK_PVH(pai);
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
pveh_p = PV_ENTRY_NULL;
pvet_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
pveh_p = pve_p;
}
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pmap_t pmap;
pt_entry_t tmplate;
boolean_t update = FALSE;
if (pve_p != PV_ENTRY_NULL)
pte_p = pve_get_ptep(pve_p);
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
if (pte_p == PT_ENTRY_NULL) {
panic("pmap_page_protect: pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, va=0x%llx ppnum: 0x%x\n",
pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)va, ppnum);
} else if ((pmap == NULL) || (atop(pte_to_pa(*pte_p)) != ppnum)) {
#if MACH_ASSERT
if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) {
pv_entry_t *check_pve_p = pveh_p;
while (check_pve_p != PV_ENTRY_NULL) {
if ((check_pve_p != pve_p) && (pve_get_ptep(check_pve_p) == pte_p)) {
panic("pmap_page_protect: duplicate pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n",
pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum);
}
check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p));
}
}
#endif
panic("pmap_page_protect: bad pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n",
pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum);
}
#if DEVELOPMENT || DEBUG
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
#else
if ((prot & VM_PROT_EXECUTE))
#endif
set_NX = FALSE;
else
set_NX = TRUE;
if (remove) {
boolean_t is_altacct = FALSE;
if (IS_ALTACCT_PAGE(pai, pve_p)) {
is_altacct = TRUE;
} else {
is_altacct = FALSE;
}
if (pte_is_wired(*pte_p)) {
pte_set_wired(pte_p, 0);
if (pmap != kernel_pmap) {
pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
}
}
if (*pte_p != ARM_PTE_TYPE_FAULT &&
pmap != kernel_pmap &&
(options & PMAP_OPTIONS_COMPRESSOR) &&
IS_INTERNAL_PAGE(pai)) {
assert(!ARM_PTE_IS_COMPRESSED(*pte_p));
tmplate = ARM_PTE_COMPRESSED;
if (is_altacct) {
tmplate |= ARM_PTE_COMPRESSED_ALT;
is_altacct = TRUE;
}
} else {
tmplate = ARM_PTE_TYPE_FAULT;
}
if ((*pte_p != ARM_PTE_TYPE_FAULT) &&
tmplate == ARM_PTE_TYPE_FAULT &&
(pmap != kernel_pmap)) {
if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt)) <= 0)
panic("pmap_page_protect_options(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
}
if (*pte_p != tmplate) {
WRITE_PTE(pte_p, tmplate);
update = TRUE;
}
pvh_cnt++;
pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count);
#if MACH_ASSERT
if (options & PMAP_OPTIONS_COMPRESSOR) {
assert(IS_INTERNAL_PAGE(pai));
}
#endif
if (pmap != kernel_pmap) {
if (IS_REUSABLE_PAGE(pai) &&
IS_INTERNAL_PAGE(pai) &&
!is_altacct) {
PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable);
OSAddAtomic(-1, &pmap->stats.reusable);
} else if (IS_INTERNAL_PAGE(pai)) {
PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal);
OSAddAtomic(-1, &pmap->stats.internal);
} else {
PMAP_STATS_ASSERTF(pmap->stats.external > 0, pmap, "stats.external %d", pmap->stats.external);
OSAddAtomic(-1, &pmap->stats.external);
}
if ((options & PMAP_OPTIONS_COMPRESSOR) &&
IS_INTERNAL_PAGE(pai)) {
OSAddAtomic64(+1, &pmap->stats.compressed);
PMAP_STATS_PEAK(pmap->stats.compressed);
pmap->stats.compressed_lifetime++;
}
if (IS_ALTACCT_PAGE(pai, pve_p)) {
assert(IS_INTERNAL_PAGE(pai));
pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
if (options & PMAP_OPTIONS_COMPRESSOR) {
pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE);
}
CLR_ALTACCT_PAGE(pai, pve_p);
} else if (IS_REUSABLE_PAGE(pai)) {
assert(IS_INTERNAL_PAGE(pai));
if (options & PMAP_OPTIONS_COMPRESSOR) {
pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
}
} else if (IS_INTERNAL_PAGE(pai)) {
pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE);
if (options & PMAP_OPTIONS_COMPRESSOR) {
pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE);
} else {
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
}
} else {
}
}
if (pve_p != PV_ENTRY_NULL) {
assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p)));
}
} else {
pt_entry_t spte;
spte = *pte_p;
if (pmap == kernel_pmap)
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
else
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO));
pte_set_ffr(tmplate, 0);
#if (__ARM_VMSA__ == 7)
if (set_NX) {
tmplate |= ARM_PTE_NX;
} else {
#if 0
tmplate &= ~ARM_PTE_NX;
#else
;
#endif
}
#else
if (set_NX)
tmplate |= ARM_PTE_NX | ARM_PTE_PNX;
else {
#if 0
if (pmap == kernel_pmap) {
tmplate &= ~ARM_PTE_PNX;
tmplate |= ARM_PTE_NX;
} else {
tmplate &= ~ARM_PTE_NX;
tmplate |= ARM_PTE_PNX;
}
#else
;
#endif
}
#endif
if (*pte_p != ARM_PTE_TYPE_FAULT &&
!ARM_PTE_IS_COMPRESSED(*pte_p) &&
*pte_p != tmplate) {
WRITE_PTE(pte_p, tmplate);
update = TRUE;
}
}
if (update)
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
pte_p = PT_ENTRY_NULL;
pvet_p = pve_p;
if (pve_p != PV_ENTRY_NULL) {
pvet_p = pve_p;
if (remove) {
assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p)));
}
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
if (remove) {
pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL);
}
UNLOCK_PVH(pai);
if (remove && (pveh_p != PV_ENTRY_NULL)) {
pv_list_free(pveh_p, pvet_p, pvh_cnt);
}
}
void
pmap_page_protect_options(
ppnum_t ppnum,
vm_prot_t prot,
unsigned int options,
__unused void *arg)
{
pmap_paddr_t phys = ptoa(ppnum);
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys))
return;
if (prot == VM_PROT_ALL) {
return;
}
PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, ppnum, prot);
pmap_page_protect_options_internal(ppnum, prot, options);
PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END);
}
bool pmap_has_prot_policy(__unused vm_prot_t prot)
{
return FALSE;
}
void
pmap_protect(
pmap_t pmap,
vm_map_address_t b,
vm_map_address_t e,
vm_prot_t prot)
{
pmap_protect_options(pmap, b, e, prot, 0, NULL);
}
static void
pmap_protect_options_internal(pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
vm_prot_t prot,
unsigned int options,
__unused void *args)
{
tt_entry_t *tte_p;
pt_entry_t *bpte_p, *epte_p;
pt_entry_t *pte_p;
boolean_t set_NX = TRUE;
#if (__ARM_VMSA__ > 7)
boolean_t set_XO = FALSE;
#endif
boolean_t should_have_removed = FALSE;
#ifndef __ARM_IC_NOALIAS_ICACHE__
boolean_t InvalidatePoU_Icache_Done = FALSE;
#endif
#if DEVELOPMENT || DEBUG
if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
if ((prot & VM_PROT_ALL) == VM_PROT_NONE) {
should_have_removed = TRUE;
}
} else
#endif
{
switch (prot) {
#if (__ARM_VMSA__ > 7)
case VM_PROT_EXECUTE:
set_XO = TRUE;
#endif
case VM_PROT_READ:
case VM_PROT_READ | VM_PROT_EXECUTE:
break;
case VM_PROT_READ | VM_PROT_WRITE:
case VM_PROT_ALL:
return;
default:
should_have_removed = TRUE;
}
}
if (should_have_removed) {
panic("%s: should have been a remove operation, "
"pmap=%p, start=%p, end=%p, prot=%#x, options=%#x, args=%p",
__FUNCTION__,
pmap, (void *)start, (void *)end, prot, options, args);
}
#if DEVELOPMENT || DEBUG
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
#else
if ((prot & VM_PROT_EXECUTE))
#endif
{
set_NX = FALSE;
} else {
set_NX = TRUE;
}
PMAP_LOCK(pmap);
tte_p = pmap_tte(pmap, start);
if ((tte_p != (tt_entry_t *) NULL) && (*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
bpte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte_p = &bpte_p[ptenum(start)];
epte_p = bpte_p + arm_atop(end - start);
pte_p = bpte_p;
for (pte_p = bpte_p;
pte_p < epte_p;
pte_p += PAGE_SIZE/ARM_PGBYTES) {
pt_entry_t spte;
#if DEVELOPMENT || DEBUG
boolean_t force_write = FALSE;
#endif
spte = *pte_p;
if ((spte == ARM_PTE_TYPE_FAULT) ||
ARM_PTE_IS_COMPRESSED(spte)) {
continue;
}
pmap_paddr_t pa;
int pai=0;
boolean_t managed=FALSE;
while (!managed) {
pa = pte_to_pa(spte);
if (!pa_valid(pa))
break;
pai = (int)pa_index(pa);
LOCK_PVH(pai);
spte = *pte_p;
pa = pte_to_pa(spte);
if (pai == (int)pa_index(pa)) {
managed =TRUE;
break; }
UNLOCK_PVH(pai);
}
if ((spte == ARM_PTE_TYPE_FAULT) ||
ARM_PTE_IS_COMPRESSED(spte)) {
continue;
}
pt_entry_t tmplate;
if (pmap == kernel_pmap) {
#if DEVELOPMENT || DEBUG
if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) {
force_write = TRUE;
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA));
} else
#endif
{
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
}
} else {
#if DEVELOPMENT || DEBUG
if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) {
force_write = TRUE;
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW));
} else
#endif
{
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO));
}
}
#if (__ARM_VMSA__ == 7)
if (set_NX)
tmplate |= ARM_PTE_NX;
else {
}
#else
if (set_NX)
tmplate |= ARM_PTE_NX | ARM_PTE_PNX;
else {
if (pmap == kernel_pmap) {
tmplate &= ~ARM_PTE_PNX;
tmplate |= ARM_PTE_NX;
} else {
tmplate |= ARM_PTE_PNX;
if (set_XO) {
tmplate &= ~ARM_PTE_APMASK;
tmplate |= ARM_PTE_AP(AP_RONA);
}
}
}
#endif
#if DEVELOPMENT || DEBUG
if (force_write) {
if (managed) {
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
tmplate |= ARM_PTE_AF;
if (IS_REFFAULT_PAGE(pai)) {
CLR_REFFAULT_PAGE(pai);
}
if (IS_MODFAULT_PAGE(pai)) {
CLR_MODFAULT_PAGE(pai);
}
}
} else if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
if (managed) {
pa_set_bits(pa, PP_ATTR_REFERENCED);
tmplate |= ARM_PTE_AF;
if (IS_REFFAULT_PAGE(pai)) {
CLR_REFFAULT_PAGE(pai);
}
}
}
#endif
pte_set_ffr(tmplate, 0);
if (((spte & ARM_PTE_NX) == ARM_PTE_NX) && (prot & VM_PROT_EXECUTE)) {
CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE);
#ifdef __ARM_IC_NOALIAS_ICACHE__
InvalidatePoU_IcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE);
#else
if (!InvalidatePoU_Icache_Done) {
InvalidatePoU_Icache();
InvalidatePoU_Icache_Done = TRUE;
}
#endif
}
WRITE_PTE_FAST(pte_p, tmplate);
if (managed) {
ASSERT_PVH_LOCKED(pai);
UNLOCK_PVH(pai);
}
}
FLUSH_PTE_RANGE(bpte_p, epte_p);
PMAP_UPDATE_TLBS(pmap, start, end);
}
PMAP_UNLOCK(pmap);
}
void
pmap_protect_options(
pmap_t pmap,
vm_map_address_t b,
vm_map_address_t e,
vm_prot_t prot,
unsigned int options,
__unused void *args)
{
vm_map_address_t l, beg;
if ((b|e) & PAGE_MASK) {
panic("pmap_protect_options() pmap %p start 0x%llx end 0x%llx\n",
pmap, (uint64_t)b, (uint64_t)e);
}
#if DEVELOPMENT || DEBUG
if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
if ((prot & VM_PROT_ALL) == VM_PROT_NONE) {
pmap_remove_options(pmap, b, e, options);
return;
}
} else
#endif
{
switch (prot) {
case VM_PROT_EXECUTE:
case VM_PROT_READ:
case VM_PROT_READ | VM_PROT_EXECUTE:
break;
case VM_PROT_READ | VM_PROT_WRITE:
case VM_PROT_ALL:
return;
default:
pmap_remove_options(pmap, b, e, options);
return;
}
}
PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(b),
VM_KERNEL_ADDRHIDE(e));
beg = b;
while (beg < e) {
l = ((beg + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK);
if (l > e)
l = e;
pmap_protect_options_internal(pmap, beg, l, prot, options, args);
beg = l;
}
PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END);
}
kern_return_t
pmap_map_block(
pmap_t pmap,
addr64_t va,
ppnum_t pa,
uint32_t size,
vm_prot_t prot,
int attr,
__unused unsigned int flags)
{
kern_return_t kr;
addr64_t original_va = va;
uint32_t page;
for (page = 0; page < size; page++) {
kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
if (kr != KERN_SUCCESS) {
panic("%s: failed pmap_enter, "
"pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x",
__FUNCTION__,
pmap, va, pa, size, prot, flags);
pmap_remove(pmap, original_va, va - original_va);
return kr;
}
va += PAGE_SIZE;
pa++;
}
return KERN_SUCCESS;
}
kern_return_t
pmap_enter(
pmap_t pmap,
vm_map_address_t v,
ppnum_t pn,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired)
{
return pmap_enter_options(pmap, v, pn, prot, fault_type, flags, wired, 0, NULL);
}
static inline void pmap_enter_pte(pmap_t pmap, pt_entry_t *pte_p, pt_entry_t pte, vm_map_address_t v)
{
if (pmap != kernel_pmap && ((pte & ARM_PTE_WIRED) != (*pte_p & ARM_PTE_WIRED)))
{
SInt16 *ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].wiredcnt);
if (pte & ARM_PTE_WIRED) {
OSAddAtomic16(1, ptd_wiredcnt_ptr);
pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
OSAddAtomic(1, (SInt32 *) &pmap->stats.wired_count);
} else {
OSAddAtomic16(-1, ptd_wiredcnt_ptr);
pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
}
}
if (*pte_p != ARM_PTE_TYPE_FAULT &&
!ARM_PTE_IS_COMPRESSED(*pte_p)) {
WRITE_PTE(pte_p, pte);
PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
} else {
WRITE_PTE(pte_p, pte);
__asm__ volatile("isb");
}
}
static pt_entry_t
wimg_to_pte(unsigned int wimg)
{
pt_entry_t pte;
switch (wimg & (VM_WIMG_MASK)) {
case VM_WIMG_IO:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_POSTED:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_WCOMB:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_WTHRU:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITETHRU);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
break;
case VM_WIMG_COPYBACK:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
break;
case VM_WIMG_INNERWBACK:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_INNERWRITEBACK);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_INNER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
break;
default:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
}
return pte;
}
static kern_return_t
pmap_enter_options_internal(
pmap_t pmap,
vm_map_address_t v,
ppnum_t pn,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options)
{
pmap_paddr_t pa = ptoa(pn);
pt_entry_t pte;
pt_entry_t spte;
pt_entry_t *pte_p;
pv_entry_t *pve_p;
boolean_t set_NX;
boolean_t set_XO = FALSE;
boolean_t refcnt_updated;
unsigned int wimg_bits;
boolean_t was_compressed, was_alt_compressed;
if ((v) & PAGE_MASK) {
panic("pmap_enter_options() pmap %p v 0x%llx\n",
pmap, (uint64_t)v);
}
if ((prot & VM_PROT_EXECUTE) && (prot & VM_PROT_WRITE) && (pmap == kernel_pmap)) {
panic("pmap_enter_options(): WX request on kernel_pmap");
}
#if DEVELOPMENT || DEBUG
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
#else
if ((prot & VM_PROT_EXECUTE))
#endif
set_NX = FALSE;
else
set_NX = TRUE;
#if (__ARM_VMSA__ > 7)
if (prot == VM_PROT_EXECUTE) {
set_XO = TRUE;
}
#endif
assert(pn != vm_page_fictitious_addr);
refcnt_updated = FALSE;
pve_p = PV_ENTRY_NULL;
was_compressed = FALSE;
was_alt_compressed = FALSE;
PMAP_LOCK(pmap);
while ((pte_p = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
PMAP_UNLOCK(pmap);
kern_return_t kr=pmap_expand(pmap, v, options, PMAP_TT_MAX_LEVEL);
if(kr) {
return kr;
}
PMAP_LOCK(pmap);
}
if (options & PMAP_OPTIONS_NOENTER) {
PMAP_UNLOCK(pmap);
return KERN_SUCCESS;
}
Pmap_enter_retry:
spte = *pte_p;
if (ARM_PTE_IS_COMPRESSED(spte)) {
assert(pmap != kernel_pmap);
OSAddAtomic64(-1, &pmap->stats.compressed);
pmap_ledger_debit(pmap, task_ledgers.internal_compressed,
PAGE_SIZE);
was_compressed = TRUE;
if (spte & ARM_PTE_COMPRESSED_ALT) {
was_alt_compressed = TRUE;
pmap_ledger_debit(
pmap,
task_ledgers.alternate_accounting_compressed,
PAGE_SIZE);
} else {
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
}
WRITE_PTE_FAST(pte_p, ARM_PTE_TYPE_FAULT);
spte = ARM_PTE_TYPE_FAULT;
refcnt_updated = TRUE;
}
if ((spte != ARM_PTE_TYPE_FAULT) && (pte_to_pa(spte) != pa)) {
pmap_remove_range(pmap, v, pte_p, pte_p + 1, 0);
PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE);
}
pte = pa_to_pte(pa) | ARM_PTE_TYPE;
if (wired && pmap != kernel_pmap)
pte |= ARM_PTE_WIRED;
#if (__ARM_VMSA__ == 7)
if (set_NX)
pte |= ARM_PTE_NX;
#else
if (set_NX)
pte |= ARM_PTE_NX | ARM_PTE_PNX;
else {
if (pmap == kernel_pmap) {
pte |= ARM_PTE_NX;
} else {
pte |= ARM_PTE_PNX;
}
}
#endif
if ((flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT)))
wimg_bits = (flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT));
else
wimg_bits = pmap_cache_attributes(pn);
pte |= wimg_to_pte(wimg_bits);
if (pmap == kernel_pmap) {
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
if (prot & VM_PROT_WRITE) {
pte |= ARM_PTE_AP(AP_RWNA);
pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED);
} else {
pte |= ARM_PTE_AP(AP_RONA);
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
#if (__ARM_VMSA__ == 7)
if ((_COMM_PAGE_BASE_ADDRESS <= v) && (v < _COMM_PAGE_BASE_ADDRESS + _COMM_PAGE_AREA_LENGTH))
pte = (pte & ~(ARM_PTE_APMASK)) | ARM_PTE_AP(AP_RORO);
#endif
} else {
if (!(pmap->nested)) {
pte |= ARM_PTE_NG;
} else if ((pmap->nested_region_asid_bitmap)
&& (v >= pmap->nested_region_subord_addr)
&& (v < (pmap->nested_region_subord_addr+pmap->nested_region_size))) {
unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT);
if ((pmap->nested_region_asid_bitmap)
&& testbit(index, (int *)pmap->nested_region_asid_bitmap))
pte |= ARM_PTE_NG;
}
#if MACH_ASSERT
if (pmap->nested_pmap != NULL) {
vm_map_address_t nest_vaddr;
pt_entry_t *nest_pte_p;
nest_vaddr = v - pmap->nested_region_grand_addr + pmap->nested_region_subord_addr;
if ((nest_vaddr >= pmap->nested_region_subord_addr)
&& (nest_vaddr < (pmap->nested_region_subord_addr+pmap->nested_region_size))
&& ((nest_pte_p = pmap_pte(pmap->nested_pmap, nest_vaddr)) != PT_ENTRY_NULL)
&& (*nest_pte_p != ARM_PTE_TYPE_FAULT)
&& (!ARM_PTE_IS_COMPRESSED(*nest_pte_p))
&& (((*nest_pte_p) & ARM_PTE_NG) != ARM_PTE_NG)) {
unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT);
if ((pmap->nested_pmap->nested_region_asid_bitmap)
&& !testbit(index, (int *)pmap->nested_pmap->nested_region_asid_bitmap)) {
panic("pmap_enter(): Global attribute conflict nest_pte_p=%p pmap=%p v=0x%llx spte=0x%llx \n",
nest_pte_p, pmap, (uint64_t)v, (uint64_t)*nest_pte_p);
}
}
}
#endif
if (prot & VM_PROT_WRITE) {
if (pa_valid(pa) && (!pa_test_bits(pa, PP_ATTR_MODIFIED))) {
if (fault_type & VM_PROT_WRITE) {
if (set_XO)
pte |= ARM_PTE_AP(AP_RWNA);
else
pte |= ARM_PTE_AP(AP_RWRW);
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
} else {
if (set_XO)
pte |= ARM_PTE_AP(AP_RONA);
else
pte |= ARM_PTE_AP(AP_RORO);
pa_set_bits(pa, PP_ATTR_REFERENCED);
pte_set_ffr(pte, 1);
}
} else {
if (set_XO)
pte |= ARM_PTE_AP(AP_RWNA);
else
pte |= ARM_PTE_AP(AP_RWRW);
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
} else {
if (set_XO)
pte |= ARM_PTE_AP(AP_RONA);
else
pte |= ARM_PTE_AP(AP_RORO);
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
}
pte |= ARM_PTE_AF;
volatile uint16_t *refcnt = NULL;
if (pmap != kernel_pmap) {
refcnt = &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt);
OSBitOrAtomic16(PT_DESC_REFCOUNT, refcnt);
if (!refcnt_updated) {
OSAddAtomic16(1, (volatile int16_t*)refcnt);
refcnt_updated = TRUE;
}
}
if (pa_valid(pa)) {
pv_entry_t **pv_h;
int pai;
boolean_t is_altacct, is_internal;
is_internal = FALSE;
is_altacct = FALSE;
pai = (int)pa_index(pa);
pv_h = pai_to_pvh(pai);
LOCK_PVH(pai);
Pmap_enter_loop:
if (pte == *pte_p) {
if (refcnt != NULL) {
assert(refcnt_updated);
if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT)
panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
}
UNLOCK_PVH(pai);
goto Pmap_enter_return;
} else if (pte_to_pa(*pte_p) == pa) {
if (refcnt != NULL) {
assert(refcnt_updated);
if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT)
panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
}
pmap_enter_pte(pmap, pte_p, pte, v);
UNLOCK_PVH(pai);
goto Pmap_enter_return;
} else if (*pte_p != ARM_PTE_TYPE_FAULT) {
UNLOCK_PVH(pai);
goto Pmap_enter_retry;
}
if (pvh_test_type(pv_h, PVH_TYPE_NULL)) {
pvh_update_head(pv_h, pte_p, PVH_TYPE_PTEP);
if (options & PMAP_OPTIONS_INTERNAL) {
SET_INTERNAL_PAGE(pai);
} else {
CLR_INTERNAL_PAGE(pai);
}
if ((options & PMAP_OPTIONS_INTERNAL) &&
(options & PMAP_OPTIONS_REUSABLE)) {
SET_REUSABLE_PAGE(pai);
} else {
CLR_REUSABLE_PAGE(pai);
}
if (pmap != kernel_pmap &&
((options & PMAP_OPTIONS_ALT_ACCT) ||
PMAP_FOOTPRINT_SUSPENDED(pmap)) &&
IS_INTERNAL_PAGE(pai)) {
SET_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
is_altacct = TRUE;
} else {
CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
}
} else {
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pt_entry_t *pte1_p;
pte1_p = pvh_ptep(pv_h);
if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) {
goto Pmap_enter_loop;
}
pve_set_ptep(pve_p, pte1_p);
pve_p->pve_next = PV_ENTRY_NULL;
if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) {
CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
SET_ALTACCT_PAGE(pai, pve_p);
}
pvh_update_head(pv_h, pve_p, PVH_TYPE_PVEP);
pve_p = PV_ENTRY_NULL;
}
if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) {
goto Pmap_enter_loop;
}
pve_set_ptep(pve_p, pte_p);
pve_p->pve_next = PV_ENTRY_NULL;
pvh_add(pv_h, pve_p);
if (pmap != kernel_pmap &&
((options & PMAP_OPTIONS_ALT_ACCT) ||
PMAP_FOOTPRINT_SUSPENDED(pmap)) &&
IS_INTERNAL_PAGE(pai)) {
SET_ALTACCT_PAGE(pai, pve_p);
is_altacct = TRUE;
}
pve_p = PV_ENTRY_NULL;
}
pmap_enter_pte(pmap, pte_p, pte, v);
if (pmap != kernel_pmap) {
if (IS_REUSABLE_PAGE(pai) &&
!is_altacct) {
assert(IS_INTERNAL_PAGE(pai));
OSAddAtomic(+1, &pmap->stats.reusable);
PMAP_STATS_PEAK(pmap->stats.reusable);
} else if (IS_INTERNAL_PAGE(pai)) {
OSAddAtomic(+1, &pmap->stats.internal);
PMAP_STATS_PEAK(pmap->stats.internal);
is_internal = TRUE;
} else {
OSAddAtomic(+1, &pmap->stats.external);
PMAP_STATS_PEAK(pmap->stats.external);
}
}
UNLOCK_PVH(pai);
if (pmap != kernel_pmap) {
pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE);
if (is_internal) {
pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE);
if (is_altacct) {
pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE);
} else {
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE);
}
}
}
OSAddAtomic(1, (SInt32 *) &pmap->stats.resident_count);
if (pmap->stats.resident_count > pmap->stats.resident_max)
pmap->stats.resident_max = pmap->stats.resident_count;
} else {
pmap_enter_pte(pmap, pte_p, pte, v);
}
Pmap_enter_return:
#if CONFIG_PGTRACE
if (pgtrace_enabled) {
for (int i = 0; i < PAGE_RATIO; i++) {
pmap_pgtrace_enter_clone(pmap, v + ARM_PGBYTES*i, 0, 0);
}
}
#endif
if (pve_p != PV_ENTRY_NULL)
pv_free(pve_p);
if (refcnt != NULL)
OSBitAndAtomic16(~PT_DESC_REFCOUNT, refcnt); PMAP_UNLOCK(pmap);
return KERN_SUCCESS;
}
kern_return_t
pmap_enter_options(
pmap_t pmap,
vm_map_address_t v,
ppnum_t pn,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options,
__unused void *arg)
{
kern_return_t kr = KERN_FAILURE;
PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v), pn, prot);
kr = pmap_enter_options_internal(pmap, v, pn, prot, fault_type, flags, wired, options);
PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr);
return kr;
}
static void
pmap_change_wiring_internal(
pmap_t pmap,
vm_map_address_t v,
boolean_t wired)
{
pt_entry_t *pte_p;
pmap_paddr_t pa;
if (pmap == kernel_pmap) {
return;
}
PMAP_LOCK(pmap);
pte_p = pmap_pte(pmap, v);
assert(pte_p != PT_ENTRY_NULL);
pa = pte_to_pa(*pte_p);
if (pa_valid(pa))
LOCK_PVH((int)pa_index(pa));
if (wired && !pte_is_wired(*pte_p)) {
pte_set_wired(pte_p, wired);
OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
} else if (!wired && pte_is_wired(*pte_p)) {
PMAP_STATS_ASSERTF(pmap->stats.wired_count >= 1, pmap, "stats.wired_count %d", pmap->stats.wired_count);
pte_set_wired(pte_p, wired);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
}
if (pa_valid(pa))
UNLOCK_PVH((int)pa_index(pa));
PMAP_UNLOCK(pmap);
}
void
pmap_change_wiring(
pmap_t pmap,
vm_map_address_t v,
boolean_t wired)
{
pmap_change_wiring_internal(pmap, v, wired);
}
static ppnum_t
pmap_find_phys_internal(
pmap_t pmap,
addr64_t va)
{
ppnum_t ppn=0;
if (pmap != kernel_pmap) {
PMAP_LOCK(pmap);
}
ppn = pmap_vtophys(pmap, va);
if (pmap != kernel_pmap) {
PMAP_UNLOCK(pmap);
}
return ppn;
}
ppnum_t
pmap_find_phys(
pmap_t pmap,
addr64_t va)
{
pmap_paddr_t pa=0;
if (pmap == kernel_pmap)
pa = mmu_kvtop(va);
else if ((current_thread()->map) && (pmap == vm_map_pmap(current_thread()->map)))
pa = mmu_uvtop(va);
if (pa) return (ppnum_t)(pa >> PAGE_SHIFT);
if (not_in_kdp) {
return pmap_find_phys_internal(pmap, va);
} else {
return pmap_vtophys(pmap, va);
}
}
pmap_paddr_t
kvtophys(
vm_offset_t va)
{
pmap_paddr_t pa;
pa = mmu_kvtop(va);
if (pa) return pa;
pa = ((pmap_paddr_t)pmap_vtophys(kernel_pmap, va)) << PAGE_SHIFT;
if (pa)
pa |= (va & PAGE_MASK);
return ((pmap_paddr_t)pa);
}
ppnum_t
pmap_vtophys(
pmap_t pmap,
addr64_t va)
{
if ((va < pmap->min) || (va >= pmap->max)) {
return 0;
}
#if (__ARM_VMSA__ == 7)
tt_entry_t *tte_p, tte;
pt_entry_t *pte_p;
ppnum_t ppn;
tte_p = pmap_tte(pmap, va);
if (tte_p == (tt_entry_t *) NULL)
return (ppnum_t) 0;
tte = *tte_p;
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(va);
ppn = (ppnum_t) atop(pte_to_pa(*pte_p) | (va & ARM_PGMASK));
#if DEVELOPMENT || DEBUG
if (ppn != 0 &&
ARM_PTE_IS_COMPRESSED(*pte_p)) {
panic("pmap_vtophys(%p,0x%llx): compressed pte_p=%p 0x%llx with ppn=0x%x\n",
pmap, va, pte_p, (uint64_t) (*pte_p), ppn);
}
#endif
} else if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK)
if ((tte & ARM_TTE_BLOCK_SUPER) == ARM_TTE_BLOCK_SUPER)
ppn = (ppnum_t) atop(suptte_to_pa(tte) | (va & ARM_TT_L1_SUPER_OFFMASK));
else
ppn = (ppnum_t) atop(sectte_to_pa(tte) | (va & ARM_TT_L1_BLOCK_OFFMASK));
else
ppn = 0;
#else
tt_entry_t *ttp;
tt_entry_t tte;
ppnum_t ppn=0;
#if __ARM64_TWO_LEVEL_PMAP__
ttp = pmap_tt2e(pmap, va);
tte = *ttp;
#else
ttp = pmap_tt1e(pmap, va);
tte = *ttp;
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID))
return (ppn);
tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, va)];
#endif
if ((tte & ARM_TTE_VALID) != (ARM_TTE_VALID))
return (ppn);
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
ppn = (ppnum_t) atop((tte & ARM_TTE_BLOCK_L2_MASK)| (va & ARM_TT_L2_OFFMASK));
return(ppn);
}
tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, va)];
ppn = (ppnum_t) atop((tte & ARM_PTE_MASK)| (va & ARM_TT_L3_OFFMASK));
#endif
return ppn;
}
static vm_offset_t
pmap_extract_internal(
pmap_t pmap,
vm_map_address_t va)
{
pmap_paddr_t pa=0;
ppnum_t ppn=0;
if (pmap == NULL) {
return 0;
}
PMAP_LOCK(pmap);
ppn = pmap_vtophys(pmap, va);
if (ppn != 0)
pa = ptoa(ppn)| ((va) & PAGE_MASK);
PMAP_UNLOCK(pmap);
return pa;
}
vm_offset_t
pmap_extract(
pmap_t pmap,
vm_map_address_t va)
{
pmap_paddr_t pa=0;
if (pmap == kernel_pmap)
pa = mmu_kvtop(va);
else if (pmap == vm_map_pmap(current_thread()->map))
pa = mmu_uvtop(va);
if (pa) return pa;
return pmap_extract_internal(pmap, va);
}
void
pmap_init_pte_page(
pmap_t pmap,
pt_entry_t *pte_p,
vm_offset_t va,
unsigned int ttlevel,
boolean_t alloc_ptd)
{
pt_desc_t *ptdp;
ptdp = *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase)));
if (ptdp == NULL) {
if (alloc_ptd) {
ptdp = ptd_alloc(pmap);
*(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase))) = ptdp;
} else {
panic("pmap_init_pte_page(): pte_p %p\n", pte_p);
}
}
pmap_init_pte_page_internal(pmap, pte_p, va, ttlevel, &ptdp);
}
void
pmap_init_pte_page_internal(
pmap_t pmap,
pt_entry_t *pte_p,
vm_offset_t va,
unsigned int ttlevel,
pt_desc_t **ptdp)
{
bzero(pte_p, ARM_PGBYTES);
__asm__ volatile("DMB ST" : : : "memory");
ptd_init(*ptdp, pmap, va, ttlevel, pte_p);
}
void
pmap_init_pte_static_page(
__unused pmap_t pmap,
pt_entry_t * pte_p,
pmap_paddr_t pa)
{
#if (__ARM_VMSA__ == 7)
unsigned int i;
pt_entry_t *pte_cur;
for (i = 0, pte_cur = pte_p;
i < (ARM_PGBYTES / sizeof(*pte_p));
i++, pa += PAGE_SIZE) {
if (pa >= avail_end) {
break;
}
*pte_cur = pa_to_pte(pa)
| ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_AP(AP_RONA)
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
pte_cur++;
}
#else
unsigned int i;
pt_entry_t *pte_cur;
pt_entry_t template;
template = ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_AP(AP_RONA) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) | ARM_PTE_NX;
for (i = 0, pte_cur = pte_p;
i < (ARM_PGBYTES / sizeof(*pte_p));
i++, pa += PAGE_SIZE) {
if (pa >= avail_end) {
break;
}
__unreachable_ok_push
if (TEST_PAGE_RATIO_4) {
*pte_cur = pa_to_pte(pa) | template;
*(pte_cur+1) = pa_to_pte(pa+0x1000) | template;
*(pte_cur+2) = pa_to_pte(pa+0x2000) | template;
*(pte_cur+3) = pa_to_pte(pa+0x3000) | template;
pte_cur += 4;
} else {
*pte_cur = pa_to_pte(pa) | template;
pte_cur++;
}
__unreachable_ok_pop
}
#endif
bzero(pte_cur, ARM_PGBYTES - ((vm_offset_t)pte_cur - (vm_offset_t)pte_p));
}
static kern_return_t
pmap_expand(
pmap_t pmap,
vm_map_address_t v,
unsigned int options,
unsigned int level)
{
#if (__ARM_VMSA__ == 7)
vm_offset_t pa;
tt_entry_t *tte_p;
tt_entry_t *tt_p;
unsigned int i;
while (tte_index(pmap, v) >= pmap->tte_index_max) {
tte_p = pmap_tt1_allocate(pmap, 2*ARM_PGBYTES, ((options & PMAP_OPTIONS_NOWAIT)? PMAP_TT_ALLOCATE_NOWAIT : 0));
if (tte_p == (tt_entry_t *)0)
return KERN_RESOURCE_SHORTAGE;
PMAP_LOCK(pmap);
if (pmap->tte_index_max > NTTES) {
pmap_tt1_deallocate(pmap, tte_p, 2*ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK);
PMAP_UNLOCK(pmap);
break;
}
simple_lock(&pmap->tt1_lock);
for (i = 0; i < pmap->tte_index_max; i++)
tte_p[i] = pmap->tte[i];
for (i = NTTES; i < 2*NTTES; i++)
tte_p[i] = ARM_TTE_TYPE_FAULT;
pmap->prev_tte = pmap->tte;
pmap->tte = tte_p;
pmap->ttep = ml_static_vtop((vm_offset_t)pmap->tte);
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) pmap->tte, 2*NTTES * sizeof(tt_entry_t));
#else
__builtin_arm_dsb(DSB_ISH);
#endif
pmap->tte_index_max = 2*NTTES;
pmap->stamp = hw_atomic_add(&pmap_stamp, 1);
for (i = 0; i < NTTES; i++)
pmap->prev_tte[i] = ARM_TTE_TYPE_FAULT;
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) pmap->prev_tte, NTTES * sizeof(tt_entry_t));
#else
__builtin_arm_dsb(DSB_ISH);
#endif
simple_unlock(&pmap->tt1_lock);
PMAP_UNLOCK(pmap);
pmap_set_pmap(pmap, current_thread());
}
if (level == 1)
return (KERN_SUCCESS);
{
tt_entry_t *tte_next_p;
PMAP_LOCK(pmap);
pa = 0;
if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
PMAP_UNLOCK(pmap);
return (KERN_SUCCESS);
}
tte_p = &pmap->tte[ttenum(v & ~ARM_TT_L1_PT_OFFMASK)];
for (i = 0, tte_next_p = tte_p; i<4; i++) {
if (tte_to_pa(*tte_next_p)) {
pa = tte_to_pa(*tte_next_p);
break;
}
tte_next_p++;
}
pa = pa & ~PAGE_MASK;
if (pa) {
tte_p = &pmap->tte[ttenum(v)];
*tte_p = pa_to_tte(pa) | (((v >> ARM_TT_L1_SHIFT) & 0x3) << 10) | ARM_TTE_TYPE_TABLE;
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) tte_p, sizeof(tt_entry_t));
#endif
PMAP_UNLOCK(pmap);
return (KERN_SUCCESS);
}
PMAP_UNLOCK(pmap);
}
v = v & ~ARM_TT_L1_PT_OFFMASK;
while (pmap_pte(pmap, v) == PT_ENTRY_NULL) {
while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if(options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
PMAP_LOCK(pmap);
if (pmap_pte(pmap, v) == PT_ENTRY_NULL) {
tt_entry_t *tte_next_p;
pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE);
pa = kvtophys((vm_offset_t)tt_p);
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE);
#endif
tte_p = &pmap->tte[ttenum(v)];
for (i = 0, tte_next_p = tte_p; i<4; i++) {
*tte_next_p = pa_to_tte(pa) | ARM_TTE_TYPE_TABLE;
tte_next_p++;
pa = pa +0x400;
}
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) tte_p, 4*sizeof(tt_entry_t));
#endif
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
}
PMAP_UNLOCK(pmap);
if (tt_p != (tt_entry_t *)NULL) {
pmap_tt_deallocate(pmap, tt_p, PMAP_TT_L2_LEVEL);
tt_p = (tt_entry_t *)NULL;
}
}
return (KERN_SUCCESS);
#else
pmap_paddr_t pa;
#if __ARM64_TWO_LEVEL_PMAP__
unsigned int ttlevel = 2;
#else
unsigned int ttlevel = 1;
#endif
tt_entry_t *tte_p;
tt_entry_t *tt_p;
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
for (; ttlevel < level; ttlevel++) {
PMAP_LOCK(pmap);
if (ttlevel == 1) {
if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) {
PMAP_UNLOCK(pmap);
while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if(options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
PMAP_LOCK(pmap);
if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) {
pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE);
pa = kvtophys((vm_offset_t)tt_p);
tte_p = pmap_tt1e( pmap, v);
*tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
if ((pmap == kernel_pmap) && (VM_MIN_KERNEL_ADDRESS < 0x00000000FFFFFFFFULL))
current_pmap()->tte[v>>ARM_TT_L1_SHIFT] = kernel_pmap->tte[v>>ARM_TT_L1_SHIFT];
}
}
} else if (ttlevel == 2) {
if (pmap_tt3e(pmap, v) == PT_ENTRY_NULL) {
PMAP_UNLOCK(pmap);
while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L3_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if(options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
PMAP_LOCK(pmap);
if ((pmap_tt3e(pmap, v) == PT_ENTRY_NULL)) {
pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v , PMAP_TT_L3_LEVEL, FALSE);
pa = kvtophys((vm_offset_t)tt_p);
tte_p = pmap_tt2e( pmap, v);
*tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
}
}
}
PMAP_UNLOCK(pmap);
if (tt_p != (tt_entry_t *)NULL) {
pmap_tt_deallocate(pmap, tt_p, ttlevel+1);
tt_p = (tt_entry_t *)NULL;
}
}
return (KERN_SUCCESS);
#endif
}
void
pmap_collect(pmap_t pmap)
{
if (pmap == PMAP_NULL)
return;
#if 0
PMAP_LOCK(pmap);
if ((pmap->nested == FALSE) && (pmap != kernel_pmap)) {
}
PMAP_UNLOCK(pmap);
#endif
return;
}
void
pmap_gc(
void)
{
pmap_t pmap, pmap_next;
boolean_t gc_wait;
if (pmap_gc_allowed &&
(pmap_gc_allowed_by_time_throttle ||
pmap_gc_forced)) {
pmap_gc_forced = FALSE;
pmap_gc_allowed_by_time_throttle = FALSE;
simple_lock(&pmaps_lock);
pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&map_pmap_list));
while (!queue_end(&map_pmap_list, (queue_entry_t)pmap)) {
if (!(pmap->gc_status & PMAP_GC_INFLIGHT))
pmap->gc_status |= PMAP_GC_INFLIGHT;
simple_unlock(&pmaps_lock);
pmap_collect(pmap);
simple_lock(&pmaps_lock);
gc_wait = (pmap->gc_status & PMAP_GC_WAIT);
pmap->gc_status &= ~(PMAP_GC_INFLIGHT|PMAP_GC_WAIT);
pmap_next = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&pmap->pmaps));
if (gc_wait) {
if (!queue_end(&map_pmap_list, (queue_entry_t)pmap_next))
pmap_next->gc_status |= PMAP_GC_INFLIGHT;
simple_unlock(&pmaps_lock);
thread_wakeup((event_t) & pmap->gc_status);
simple_lock(&pmaps_lock);
}
pmap = pmap_next;
}
simple_unlock(&pmaps_lock);
}
}
void
pmap_release_pages_fast(void)
{
}
void
compute_pmap_gc_throttle(
void *arg __unused)
{
pmap_gc_allowed_by_time_throttle = TRUE;
}
kern_return_t
pmap_attribute_cache_sync(
ppnum_t pp,
vm_size_t size,
__unused vm_machine_attribute_t attribute,
__unused vm_machine_attribute_val_t * value)
{
if (size > PAGE_SIZE) {
panic("pmap_attribute_cache_sync size: 0x%llx\n", (uint64_t)size);
} else
cache_sync_page(pp);
return KERN_SUCCESS;
}
void
pmap_sync_page_data_phys(
ppnum_t pp)
{
cache_sync_page(pp);
}
void
pmap_sync_page_attributes_phys(
ppnum_t pp)
{
flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE);
}
#if CONFIG_COREDUMP
boolean_t
coredumpok(
vm_map_t map,
vm_offset_t va)
{
pt_entry_t *pte_p;
pt_entry_t spte;
pte_p = pmap_pte(map->pmap, va);
if (0 == pte_p)
return FALSE;
spte = *pte_p;
return ((spte & ARM_PTE_ATTRINDXMASK) == ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT));
}
#endif
void
fillPage(
ppnum_t pn,
unsigned int fill)
{
unsigned int *addr;
int count;
addr = (unsigned int *) phystokv(ptoa(pn));
count = PAGE_SIZE / sizeof(unsigned int);
while (count--)
*addr++ = fill;
}
extern void mapping_set_mod(ppnum_t pn);
void
mapping_set_mod(
ppnum_t pn)
{
pmap_set_modify(pn);
}
extern void mapping_set_ref(ppnum_t pn);
void
mapping_set_ref(
ppnum_t pn)
{
pmap_set_reference(pn);
}
static void
phys_attribute_clear_internal(
ppnum_t pn,
unsigned int bits,
int options,
void *arg)
{
pmap_paddr_t pa = ptoa(pn);
vm_prot_t allow_mode = VM_PROT_ALL;
if ((bits & PP_ATTR_MODIFIED) &&
(options & PMAP_OPTIONS_NOFLUSH) &&
(arg == NULL)) {
panic("phys_attribute_clear(0x%x,0x%x,0x%x,%p): "
"should not clear 'modified' without flushing TLBs\n",
pn, bits, options, arg);
}
assert(pn != vm_page_fictitious_addr);
if (bits & PP_ATTR_REFERENCED)
allow_mode &= ~(VM_PROT_READ | VM_PROT_EXECUTE);
if (bits & PP_ATTR_MODIFIED)
allow_mode &= ~VM_PROT_WRITE;
if (bits == PP_ATTR_NOENCRYPT) {
pa_clear_bits(pa, bits);
return;
}
if (arm_force_fast_fault_internal(pn, allow_mode, options))
pa_clear_bits(pa, bits);
return;
}
static void
phys_attribute_clear(
ppnum_t pn,
unsigned int bits,
int options,
void *arg)
{
PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits);
phys_attribute_clear_internal(pn, bits, options, arg);
PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END);
}
static void
phys_attribute_set_internal(
ppnum_t pn,
unsigned int bits)
{
pmap_paddr_t pa = ptoa(pn);
assert(pn != vm_page_fictitious_addr);
pa_set_bits(pa, bits);
return;
}
static void
phys_attribute_set(
ppnum_t pn,
unsigned int bits)
{
phys_attribute_set_internal(pn, bits);
}
static boolean_t
phys_attribute_test(
ppnum_t pn,
unsigned int bits)
{
pmap_paddr_t pa = ptoa(pn);
assert(pn != vm_page_fictitious_addr);
return pa_test_bits(pa, bits);
}
void
pmap_set_modify(ppnum_t pn)
{
phys_attribute_set(pn, PP_ATTR_MODIFIED);
}
void
pmap_clear_modify(
ppnum_t pn)
{
phys_attribute_clear(pn, PP_ATTR_MODIFIED, 0, NULL);
}
boolean_t
pmap_is_modified(
ppnum_t pn)
{
return phys_attribute_test(pn, PP_ATTR_MODIFIED);
}
static void
pmap_set_reference(
ppnum_t pn)
{
phys_attribute_set(pn, PP_ATTR_REFERENCED);
}
void
pmap_clear_reference(
ppnum_t pn)
{
phys_attribute_clear(pn, PP_ATTR_REFERENCED, 0, NULL);
}
boolean_t
pmap_is_referenced(
ppnum_t pn)
{
return phys_attribute_test(pn, PP_ATTR_REFERENCED);
}
unsigned int
pmap_get_refmod(
ppnum_t pn)
{
return (((phys_attribute_test(pn, PP_ATTR_MODIFIED)) ? VM_MEM_MODIFIED : 0)
| ((phys_attribute_test(pn, PP_ATTR_REFERENCED)) ? VM_MEM_REFERENCED : 0));
}
void
pmap_clear_refmod_options(
ppnum_t pn,
unsigned int mask,
unsigned int options,
void *arg)
{
unsigned int bits;
bits = ((mask & VM_MEM_MODIFIED) ? PP_ATTR_MODIFIED : 0) |
((mask & VM_MEM_REFERENCED) ? PP_ATTR_REFERENCED : 0);
phys_attribute_clear(pn, bits, options, arg);
}
void
pmap_clear_refmod(
ppnum_t pn,
unsigned int mask)
{
pmap_clear_refmod_options(pn, mask, 0, NULL);
}
unsigned int
pmap_disconnect_options(
ppnum_t pn,
unsigned int options,
void *arg)
{
if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED)) {
if (pmap_is_modified(pn)) {
options |= PMAP_OPTIONS_COMPRESSOR;
} else {
}
}
pmap_page_protect_options(pn, 0, options, arg);
return (pmap_get_refmod(pn));
}
unsigned int
pmap_disconnect(
ppnum_t pn)
{
pmap_page_protect(pn, 0);
return (pmap_get_refmod(pn));
}
boolean_t
pmap_has_managed_page(ppnum_t first, ppnum_t last)
{
if (ptoa(first) >= vm_last_phys) return (FALSE);
if (ptoa(last) < vm_first_phys) return (FALSE);
return (TRUE);
}
boolean_t
pmap_is_noencrypt(
ppnum_t pn)
{
#if DEVELOPMENT || DEBUG
boolean_t result = FALSE;
if (!pa_valid(ptoa(pn))) return FALSE;
result = (phys_attribute_test(pn, PP_ATTR_NOENCRYPT));
return result;
#else
#pragma unused(pn)
return FALSE;
#endif
}
void
pmap_set_noencrypt(
ppnum_t pn)
{
#if DEVELOPMENT || DEBUG
if (!pa_valid(ptoa(pn))) return;
phys_attribute_set(pn, PP_ATTR_NOENCRYPT);
#else
#pragma unused(pn)
#endif
}
void
pmap_clear_noencrypt(
ppnum_t pn)
{
#if DEVELOPMENT || DEBUG
if (!pa_valid(ptoa(pn))) return;
phys_attribute_clear(pn, PP_ATTR_NOENCRYPT, 0, NULL);
#else
#pragma unused(pn)
#endif
}
void
pmap_lock_phys_page(ppnum_t pn)
{
int pai;
pmap_paddr_t phys = ptoa(pn);
if (pa_valid(phys)) {
pai = (int)pa_index(phys);
LOCK_PVH(pai);
} else
simple_lock(&phys_backup_lock);
}
void
pmap_unlock_phys_page(ppnum_t pn)
{
int pai;
pmap_paddr_t phys = ptoa(pn);
if (pa_valid(phys)) {
pai = (int)pa_index(phys);
UNLOCK_PVH(pai);
} else
simple_unlock(&phys_backup_lock);
}
static void
pmap_switch_user_ttb_internal(
pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
pmap_cpu_data_t *cpu_data_ptr;
cpu_data_ptr = pmap_get_cpu_data();
if ((cpu_data_ptr->cpu_user_pmap != PMAP_NULL)
&& (cpu_data_ptr->cpu_user_pmap != kernel_pmap)) {
unsigned int c;
c = hw_atomic_sub((volatile uint32_t *)&cpu_data_ptr->cpu_user_pmap->cpu_ref, 1);
if ((c == 0) && (cpu_data_ptr->cpu_user_pmap->prev_tte != 0)) {
tt_entry_t *tt_entry;
tt_entry = cpu_data_ptr->cpu_user_pmap->prev_tte;
cpu_data_ptr->cpu_user_pmap->prev_tte = (tt_entry_t *) NULL;
pmap_tt1_deallocate(cpu_data_ptr->cpu_user_pmap, tt_entry, ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK);
}
}
cpu_data_ptr->cpu_user_pmap = pmap;
cpu_data_ptr->cpu_user_pmap_stamp = pmap->stamp;
(void) hw_atomic_add((volatile uint32_t *)&pmap->cpu_ref, 1);
#if MACH_ASSERT && __ARM_USER_PROTECT__
{
unsigned int ttbr0_val, ttbr1_val;
__asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
__asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
if (ttbr0_val != ttbr1_val) {
panic("Misaligned ttbr0 %08X\n", ttbr0_val);
}
}
#endif
if (pmap->tte_index_max == NTTES) {
__asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(2));
__asm__ volatile("isb");
#if !__ARM_USER_PROTECT__
set_mmu_ttb(pmap->ttep);
#endif
} else {
#if !__ARM_USER_PROTECT__
set_mmu_ttb(pmap->ttep);
#endif
__asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(1));
__asm__ volatile("isb");
#if MACH_ASSERT && __ARM_USER_PROTECT__
if (pmap->ttep & 0x1000) {
panic("Misaligned ttbr0 %08X\n", pmap->ttep);
}
#endif
}
#if !__ARM_USER_PROTECT__
set_context_id(pmap->asid);
#endif
#else
pmap_get_cpu_data()->cpu_user_pmap = pmap;
pmap_get_cpu_data()->cpu_user_pmap_stamp = pmap->stamp;
#if !__arm64__
set_context_id(pmap->asid);
#endif
if (pmap == kernel_pmap) {
set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK);
} else {
set_mmu_ttb((pmap->ttep & TTBR_BADDR_MASK)|(((uint64_t)pmap->asid) << TTBR_ASID_SHIFT));
}
#endif
}
void
pmap_switch_user_ttb(
pmap_t pmap)
{
pmap_switch_user_ttb_internal(pmap);
}
#if (__ARM_VMSA__ == 7)
boolean_t
arm_swap_readable_type(
vm_map_address_t addr,
unsigned int spsr)
{
int ap;
pt_entry_t spte;
pt_entry_t *ptep;
ptep = pmap_pte(current_pmap(), addr);
if (ptep == PT_ENTRY_NULL)
return (FALSE);
spte = *ptep;
if (spte == ARM_PTE_TYPE_FAULT ||
ARM_PTE_IS_COMPRESSED(spte))
return (FALSE);
ap = (spte & ARM_PTE_APMASK);
if (spsr & 0xf) {
panic("arm_swap_readable_type supv");
return TRUE;
} else {
if ((ap == ARM_PTE_AP(AP_RWRW)) || (ap == ARM_PTE_AP(AP_RORO)))
return (FALSE);
else
return (TRUE);
}
}
#endif
static boolean_t
arm_force_fast_fault_internal(
ppnum_t ppnum,
vm_prot_t allow_mode,
int options)
{
pmap_paddr_t phys = ptoa(ppnum);
pv_entry_t *pve_p;
pt_entry_t *pte_p;
int pai;
boolean_t result;
pv_entry_t **pv_h;
boolean_t is_reusable, is_internal;
boolean_t ref_fault;
boolean_t mod_fault;
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return FALSE;
}
result = TRUE;
ref_fault = FALSE;
mod_fault = FALSE;
pai = (int)pa_index(phys);
LOCK_PVH(pai);
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
}
is_reusable = IS_REUSABLE_PAGE(pai);
is_internal = IS_INTERNAL_PAGE(pai);
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pt_entry_t spte;
pt_entry_t tmplate;
pmap_t pmap;
boolean_t update_pte;
if (pve_p != PV_ENTRY_NULL)
pte_p = pve_get_ptep(pve_p);
if (pte_p == PT_ENTRY_NULL) {
panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum);
}
if (*pte_p == ARM_PTE_EMPTY) {
panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
}
if (ARM_PTE_IS_COMPRESSED(*pte_p)) {
panic("pte is COMPRESSED: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
}
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
assert(va >= pmap->min && va < pmap->max);
if (pte_is_wired(*pte_p) || pmap == kernel_pmap) {
result = FALSE;
break;
}
spte = *pte_p;
tmplate = spte;
update_pte = FALSE;
if ((allow_mode & VM_PROT_READ) != VM_PROT_READ) {
tmplate = tmplate & ~ARM_PTE_AF;
update_pte = TRUE;
ref_fault = TRUE;
}
if ((allow_mode & VM_PROT_WRITE) != VM_PROT_WRITE) {
if (pmap == kernel_pmap) {
if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWNA)) {
tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
}
} else {
if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWRW)) {
tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO));
}
}
pte_set_ffr(tmplate, 1);
update_pte = TRUE;
mod_fault = TRUE;
}
if (update_pte) {
if (*pte_p != ARM_PTE_TYPE_FAULT &&
!ARM_PTE_IS_COMPRESSED(*pte_p)) {
WRITE_PTE(pte_p, tmplate);
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
} else {
WRITE_PTE(pte_p, tmplate);
__asm__ volatile("isb");
}
}
if (IS_ALTACCT_PAGE(pai, pve_p)) {
} else if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
is_reusable &&
is_internal &&
pmap != kernel_pmap) {
PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable);
OSAddAtomic(-1, &pmap->stats.reusable);
OSAddAtomic(+1, &pmap->stats.internal);
PMAP_STATS_PEAK(pmap->stats.internal);
PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal);
pmap_ledger_credit(pmap,
task_ledgers.internal,
machine_ptob(1));
assert(!IS_ALTACCT_PAGE(pai, pve_p));
assert(IS_INTERNAL_PAGE(pai));
pmap_ledger_credit(pmap,
task_ledgers.phys_footprint,
machine_ptob(1));
{
arm_clear_fast_fault(ppnum, VM_PROT_WRITE);
}
} else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
!is_reusable &&
is_internal &&
pmap != kernel_pmap) {
OSAddAtomic(+1, &pmap->stats.reusable);
PMAP_STATS_PEAK(pmap->stats.reusable);
PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable);
PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal);
OSAddAtomic(-1, &pmap->stats.internal);
pmap_ledger_debit(pmap,
task_ledgers.internal,
machine_ptob(1));
assert(!IS_ALTACCT_PAGE(pai, pve_p));
assert(IS_INTERNAL_PAGE(pai));
pmap_ledger_debit(pmap,
task_ledgers.phys_footprint,
machine_ptob(1));
}
pte_p = PT_ENTRY_NULL;
if (pve_p != PV_ENTRY_NULL)
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
if (is_internal) {
if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
is_reusable) {
CLR_REUSABLE_PAGE(pai);
} else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
!is_reusable) {
SET_REUSABLE_PAGE(pai);
}
}
if (mod_fault) {
SET_MODFAULT_PAGE(pai);
}
if (ref_fault) {
SET_REFFAULT_PAGE(pai);
}
UNLOCK_PVH(pai);
return result;
}
boolean_t
arm_force_fast_fault(
ppnum_t ppnum,
vm_prot_t allow_mode,
int options,
__unused void *arg)
{
pmap_paddr_t phys = ptoa(ppnum);
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return FALSE;
}
return arm_force_fast_fault_internal(ppnum, allow_mode, options);
}
boolean_t
arm_clear_fast_fault(
ppnum_t ppnum,
vm_prot_t fault_type)
{
pmap_paddr_t pa = ptoa(ppnum);
pv_entry_t *pve_p;
pt_entry_t *pte_p;
int pai;
boolean_t result;
pv_entry_t **pv_h;
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(pa)) {
return FALSE;
}
result = FALSE;
pai = (int)pa_index(pa);
ASSERT_PVH_LOCKED(pai);
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
}
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pt_entry_t spte;
pt_entry_t tmplate;
pmap_t pmap;
if (pve_p != PV_ENTRY_NULL)
pte_p = pve_get_ptep(pve_p);
if (pte_p == PT_ENTRY_NULL) {
panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum);
}
if (*pte_p == ARM_PTE_EMPTY) {
panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
}
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
assert(va >= pmap->min && va < pmap->max);
spte = *pte_p;
tmplate = spte;
if ((fault_type & VM_PROT_WRITE) && (pte_is_ffr(spte))) {
{
if (pmap == kernel_pmap)
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA));
else
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW));
}
tmplate |= ARM_PTE_AF;
pte_set_ffr(tmplate, 0);
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
} else if ((fault_type & VM_PROT_READ) && ((spte & ARM_PTE_AF) != ARM_PTE_AF)) {
tmplate = spte | ARM_PTE_AF;
{
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
}
if (spte != tmplate) {
if (spte != ARM_PTE_TYPE_FAULT) {
WRITE_PTE(pte_p, tmplate);
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
} else {
WRITE_PTE(pte_p, tmplate);
__asm__ volatile("isb");
}
result = TRUE;
}
pte_p = PT_ENTRY_NULL;
if (pve_p != PV_ENTRY_NULL)
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
return result;
}
static kern_return_t
arm_fast_fault_internal(
pmap_t pmap,
vm_map_address_t va,
vm_prot_t fault_type,
__unused boolean_t from_user)
{
kern_return_t result = KERN_FAILURE;
pt_entry_t *ptep;
pt_entry_t spte = ARM_PTE_TYPE_FAULT;
int pai;
pmap_paddr_t pa;
PMAP_LOCK(pmap);
ptep = pmap_pte(pmap, va);
if (ptep != PT_ENTRY_NULL) {
spte = *ptep;
pa = pte_to_pa(spte);
if ((spte == ARM_PTE_TYPE_FAULT) ||
ARM_PTE_IS_COMPRESSED(spte) ||
(!pa_valid(pa))) {
PMAP_UNLOCK(pmap);
return result;
}
pai = (int)pa_index(pa);
LOCK_PVH(pai);
} else {
PMAP_UNLOCK(pmap);
return result;
}
if ((IS_REFFAULT_PAGE(pai)) ||
((fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai))) {
if (IS_REFFAULT_PAGE(pai)) {
CLR_REFFAULT_PAGE(pai);
}
if ( (fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai)) {
CLR_MODFAULT_PAGE(pai);
}
if (arm_clear_fast_fault((ppnum_t)atop(pa),fault_type)) {
result = KERN_SUCCESS;
}
}
UNLOCK_PVH(pai);
PMAP_UNLOCK(pmap);
return result;
}
kern_return_t
arm_fast_fault(
pmap_t pmap,
vm_map_address_t va,
vm_prot_t fault_type,
__unused boolean_t from_user)
{
kern_return_t result = KERN_FAILURE;
if (va < pmap->min || va >= pmap->max)
return result;
PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(va), fault_type,
from_user);
#if (__ARM_VMSA__ == 7)
if (pmap != kernel_pmap) {
pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data();
pmap_t cur_pmap;
pmap_t cur_user_pmap;
cur_pmap = current_pmap();
cur_user_pmap = cpu_data_ptr->cpu_user_pmap;
if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) {
if (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp) {
pmap_set_pmap(pmap, current_thread());
result = KERN_SUCCESS;
goto done;
}
}
}
#endif
result = arm_fast_fault_internal(pmap, va, fault_type, from_user);
#if (__ARM_VMSA__ == 7)
done:
#endif
PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_END, result);
return result;
}
void
pmap_copy_page(
ppnum_t psrc,
ppnum_t pdst)
{
bcopy_phys((addr64_t) (ptoa(psrc)),
(addr64_t) (ptoa(pdst)),
PAGE_SIZE);
}
void
pmap_copy_part_page(
ppnum_t psrc,
vm_offset_t src_offset,
ppnum_t pdst,
vm_offset_t dst_offset,
vm_size_t len)
{
bcopy_phys((addr64_t) (ptoa(psrc) + src_offset),
(addr64_t) (ptoa(pdst) + dst_offset),
len);
}
void
pmap_zero_page(
ppnum_t pn)
{
assert(pn != vm_page_fictitious_addr);
bzero_phys((addr64_t) ptoa(pn), PAGE_SIZE);
}
void
pmap_zero_part_page(
ppnum_t pn,
vm_offset_t offset,
vm_size_t len)
{
assert(pn != vm_page_fictitious_addr);
assert(offset + len <= PAGE_SIZE);
bzero_phys((addr64_t) (ptoa(pn) + offset), len);
}
void
inval_copy_windows(
__unused thread_t t)
{
}
void
pmap_map_globals(
void)
{
pt_entry_t *ptep, pte;
ptep = pmap_pte(kernel_pmap, LOWGLOBAL_ALIAS);
assert(ptep != PT_ENTRY_NULL);
assert(*ptep == ARM_PTE_EMPTY);
pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE;
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
*ptep = pte;
FLUSH_PTE_RANGE(ptep,(ptep+1));
PMAP_UPDATE_TLBS(kernel_pmap, LOWGLOBAL_ALIAS, LOWGLOBAL_ALIAS + PAGE_SIZE);
}
vm_offset_t
pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index)
{
return (vm_offset_t)(CPUWINDOWS_BASE + (PAGE_SIZE * ((CPUWINDOWS_MAX * cpu_num) + index)));
}
static unsigned int
pmap_map_cpu_windows_copy_internal(
ppnum_t pn,
vm_prot_t prot,
unsigned int wimg_bits)
{
pt_entry_t *ptep = NULL, pte;
unsigned int cpu_num;
unsigned int i;
vm_offset_t cpu_copywindow_vaddr = 0;
cpu_num = pmap_get_cpu_data()->cpu_number;
for (i = 0; i<CPUWINDOWS_MAX; i++) {
cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, i);
ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr);
assert(!ARM_PTE_IS_COMPRESSED(*ptep));
if (*ptep == ARM_PTE_TYPE_FAULT)
break;
}
if (i == CPUWINDOWS_MAX) {
panic("pmap_map_cpu_windows_copy: out of window\n");
}
pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX;
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
pte |= wimg_to_pte(wimg_bits);
if (prot & VM_PROT_WRITE) {
pte |= ARM_PTE_AP(AP_RWNA);
} else {
pte |= ARM_PTE_AP(AP_RONA);
}
WRITE_PTE(ptep, pte);
PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE);
return(i);
}
unsigned int
pmap_map_cpu_windows_copy(
ppnum_t pn,
vm_prot_t prot,
unsigned int wimg_bits)
{
return pmap_map_cpu_windows_copy_internal(pn, prot, wimg_bits);
}
static void
pmap_unmap_cpu_windows_copy_internal(
unsigned int index)
{
pt_entry_t *ptep;
unsigned int cpu_num;
vm_offset_t cpu_copywindow_vaddr = 0;
cpu_num = pmap_get_cpu_data()->cpu_number;
cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, index);
__asm__ volatile("dsb sy");
ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr);
WRITE_PTE(ptep, ARM_PTE_TYPE_FAULT);
PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE);
}
void
pmap_unmap_cpu_windows_copy(
unsigned int index)
{
return pmap_unmap_cpu_windows_copy_internal(index);
}
static void
pmap_set_nested_internal(
pmap_t pmap)
{
pmap->nested = TRUE;
}
void
pmap_set_nested(
pmap_t pmap)
{
pmap_set_nested_internal(pmap);
}
static kern_return_t
pmap_nest_internal(
pmap_t grand,
pmap_t subord,
addr64_t vstart,
addr64_t nstart,
uint64_t size)
{
kern_return_t kr = KERN_FAILURE;
vm_map_offset_t vaddr, nvaddr;
tt_entry_t *stte_p;
tt_entry_t *gtte_p;
unsigned int i;
unsigned int num_tte;
unsigned int nested_region_asid_bitmap_size;
unsigned int* nested_region_asid_bitmap;
int expand_options = 0;
#if (__ARM_VMSA__ == 7)
if (((size|vstart|nstart) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) {
return KERN_INVALID_VALUE;
}
#else
if (((size|vstart|nstart) & (ARM_TT_L2_OFFMASK)) != 0x0ULL) {
panic("pmap_nest() pmap %p has a nested pmap 0x%llx, 0x%llx, 0x%llx\n", grand, vstart, nstart, size);
}
#endif
if ((grand->nested_pmap != PMAP_NULL) && (grand->nested_pmap != subord)) {
panic("pmap_nest() pmap %p has a nested pmap\n", grand);
}
if (subord->nested_region_asid_bitmap == NULL) {
nested_region_asid_bitmap_size = (unsigned int)(size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY);
nested_region_asid_bitmap = kalloc(nested_region_asid_bitmap_size*sizeof(unsigned int));
bzero(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int));
PMAP_LOCK(subord);
if (subord->nested_region_asid_bitmap == NULL) {
subord->nested_region_asid_bitmap = nested_region_asid_bitmap;
subord->nested_region_asid_bitmap_size = nested_region_asid_bitmap_size;
subord->nested_region_subord_addr = nstart;
subord->nested_region_size = (mach_vm_offset_t) size;
nested_region_asid_bitmap = NULL;
}
PMAP_UNLOCK(subord);
if (nested_region_asid_bitmap != NULL) {
kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int));
}
}
if ((subord->nested_region_subord_addr + subord->nested_region_size) < (nstart+size)) {
uint64_t new_size;
unsigned int new_nested_region_asid_bitmap_size;
unsigned int* new_nested_region_asid_bitmap;
nested_region_asid_bitmap = NULL;
nested_region_asid_bitmap_size = 0;
new_size = nstart + size - subord->nested_region_subord_addr;
new_nested_region_asid_bitmap_size = (unsigned int)((new_size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY)) + 1;
new_nested_region_asid_bitmap = kalloc(new_nested_region_asid_bitmap_size*sizeof(unsigned int));
PMAP_LOCK(subord);
if (subord->nested_region_size < new_size) {
bzero(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int));
bcopy(subord->nested_region_asid_bitmap, new_nested_region_asid_bitmap, subord->nested_region_asid_bitmap_size);
nested_region_asid_bitmap_size = subord->nested_region_asid_bitmap_size;
nested_region_asid_bitmap = subord->nested_region_asid_bitmap;
subord->nested_region_asid_bitmap = new_nested_region_asid_bitmap;
subord->nested_region_asid_bitmap_size = new_nested_region_asid_bitmap_size;
subord->nested_region_size = new_size;
new_nested_region_asid_bitmap = NULL;
}
PMAP_UNLOCK(subord);
if (nested_region_asid_bitmap != NULL)
kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int));
if (new_nested_region_asid_bitmap != NULL)
kfree(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int));
}
PMAP_LOCK(subord);
if (grand->nested_pmap == PMAP_NULL) {
grand->nested_pmap = subord;
grand->nested_region_grand_addr = vstart;
grand->nested_region_subord_addr = nstart;
grand->nested_region_size = (mach_vm_offset_t) size;
} else {
if ((grand->nested_region_grand_addr > vstart)) {
panic("pmap_nest() pmap %p : attempt to nest outside the nested region\n", grand);
}
else if ((grand->nested_region_grand_addr + grand->nested_region_size) < (vstart+size)) {
grand->nested_region_size = (mach_vm_offset_t)(vstart - grand->nested_region_grand_addr + size);
}
}
#if (__ARM_VMSA__ == 7)
nvaddr = (vm_map_offset_t) nstart;
vaddr = (vm_map_offset_t) vstart;
num_tte = size >> ARM_TT_L1_SHIFT;
for (i = 0; i < num_tte; i++) {
stte_p = pmap_tte(subord, nvaddr);
if ((stte_p == (tt_entry_t *)NULL) || (((*stte_p) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)) {
PMAP_UNLOCK(subord);
kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L2_LEVEL);
if (kr != KERN_SUCCESS) {
PMAP_LOCK(grand);
goto done;
}
PMAP_LOCK(subord);
}
PMAP_UNLOCK(subord);
PMAP_LOCK(grand);
stte_p = pmap_tte(grand, vaddr);
if (stte_p == (tt_entry_t *)NULL) {
PMAP_UNLOCK(grand);
kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L1_LEVEL);
if (kr != KERN_SUCCESS) {
PMAP_LOCK(grand);
goto done;
}
} else {
PMAP_UNLOCK(grand);
kr = KERN_SUCCESS;
}
PMAP_LOCK(subord);
nvaddr += ARM_TT_L1_SIZE;
vaddr += ARM_TT_L1_SIZE;
}
#else
nvaddr = (vm_map_offset_t) nstart;
num_tte = (unsigned int)(size >> ARM_TT_L2_SHIFT);
for (i = 0; i < num_tte; i++) {
stte_p = pmap_tt2e(subord, nvaddr);
if (stte_p == PT_ENTRY_NULL || *stte_p == ARM_TTE_EMPTY) {
PMAP_UNLOCK(subord);
kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L3_LEVEL);
if (kr != KERN_SUCCESS) {
PMAP_LOCK(grand);
goto done;
}
PMAP_LOCK(subord);
}
nvaddr += ARM_TT_L2_SIZE;
}
#endif
PMAP_UNLOCK(subord);
PMAP_LOCK(grand);
nvaddr = (vm_map_offset_t) nstart;
vaddr = (vm_map_offset_t) vstart;
#if (__ARM_VMSA__ == 7)
for (i = 0; i < num_tte; i++) {
stte_p = pmap_tte(subord, nvaddr);
gtte_p = pmap_tte(grand, vaddr);
*gtte_p = *stte_p;
nvaddr += ARM_TT_L1_SIZE;
vaddr += ARM_TT_L1_SIZE;
}
#else
for (i = 0; i < num_tte; i++) {
stte_p = pmap_tt2e(subord, nstart);
gtte_p = pmap_tt2e(grand, vaddr);
if (gtte_p == PT_ENTRY_NULL) {
PMAP_UNLOCK(grand);
kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L2_LEVEL);
PMAP_LOCK(grand);
if (kr != KERN_SUCCESS) {
goto done;
}
gtte_p = pmap_tt2e(grand, vaddr);
}
*gtte_p = *stte_p;
vaddr += ARM_TT_L2_SIZE;
nstart += ARM_TT_L2_SIZE;
}
#endif
kr = KERN_SUCCESS;
done:
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, vstart), num_tte * sizeof(tt_entry_t));
#endif
#if (__ARM_VMSA__ > 7)
assert((size & 0xFFFFFFFF00000000ULL) == 0);
#endif
PMAP_UPDATE_TLBS(grand, vstart, vstart + size);
PMAP_UNLOCK(grand);
return kr;
}
kern_return_t pmap_nest(
pmap_t grand,
pmap_t subord,
addr64_t vstart,
addr64_t nstart,
uint64_t size)
{
kern_return_t kr = KERN_FAILURE;
PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord),
VM_KERNEL_ADDRHIDE(vstart));
kr = pmap_nest_internal(grand, subord, vstart, nstart, size);
PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, kr);
return kr;
}
kern_return_t
pmap_unnest(
pmap_t grand,
addr64_t vaddr,
uint64_t size)
{
return(pmap_unnest_options(grand, vaddr, size, 0));
}
static kern_return_t
pmap_unnest_options_internal(
pmap_t grand,
addr64_t vaddr,
uint64_t size,
unsigned int option)
{
vm_map_offset_t start;
vm_map_offset_t addr;
tt_entry_t *tte_p;
unsigned int current_index;
unsigned int start_index;
unsigned int max_index;
unsigned int num_tte;
unsigned int i;
#if (__ARM_VMSA__ == 7)
if (((size|vaddr) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) {
panic("pmap_unnest(): unaligned request\n");
}
#else
if (((size|vaddr) & ARM_TT_L2_OFFMASK) != 0x0ULL) {
panic("pmap_unnest(): unaligned request\n");
}
#endif
if ((option & PMAP_UNNEST_CLEAN) == 0)
{
PMAP_LOCK(grand->nested_pmap);
start = vaddr - grand->nested_region_grand_addr + grand->nested_region_subord_addr ;
start_index = (unsigned int)((vaddr - grand->nested_region_grand_addr) >> ARM_TT_TWIG_SHIFT);
max_index = (unsigned int)(start_index + (size >> ARM_TT_TWIG_SHIFT));
num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT);
if (size > grand->nested_region_size) {
panic("pmap_unnest() pmap %p %llu, %llu\n", grand, size, (uint64_t)grand->nested_region_size);
}
for (current_index = start_index, addr = start; current_index < max_index; current_index++) {
pt_entry_t *bpte, *epte, *cpte;
if(!testbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap)) {
setbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap);
bpte = pmap_pte(grand->nested_pmap, addr);
epte = bpte + (ARM_TT_LEAF_INDEX_MASK>>ARM_TT_LEAF_SHIFT);
for (cpte = bpte; cpte <= epte; cpte++) {
pmap_paddr_t pa;
int pai=0;
boolean_t managed=FALSE;
pt_entry_t spte;
if ((*cpte != ARM_PTE_TYPE_FAULT)
&& (!ARM_PTE_IS_COMPRESSED(*cpte))) {
spte = *cpte;
while (!managed) {
pa = pte_to_pa(spte);
if (!pa_valid(pa))
break;
pai = (int)pa_index(pa);
LOCK_PVH(pai);
spte = *cpte;
pa = pte_to_pa(spte);
if (pai == (int)pa_index(pa)) {
managed =TRUE;
break; }
UNLOCK_PVH(pai);
}
if (((spte & ARM_PTE_NG) != ARM_PTE_NG)) {
WRITE_PTE(cpte, (spte | ARM_PTE_NG));
}
if (managed)
{
ASSERT_PVH_LOCKED(pai);
UNLOCK_PVH(pai);
}
}
}
}
addr += ARM_TT_TWIG_SIZE;
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) pmap_pte(grand->nested_pmap, start), num_tte * sizeof(tt_entry_t));
#endif
PMAP_UPDATE_TLBS(grand->nested_pmap, start, start + size);
}
PMAP_UNLOCK(grand->nested_pmap);
}
PMAP_LOCK(grand);
start = vaddr;
addr = vaddr;
num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT);
for (i = 0; i < num_tte; i++) {
tte_p = pmap_tte(grand, addr);
*tte_p = ARM_TTE_TYPE_FAULT;
addr += ARM_TT_TWIG_SIZE;
}
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, start), num_tte * sizeof(tt_entry_t));
#endif
PMAP_UPDATE_TLBS(grand, start, start + size);
PMAP_UNLOCK(grand);
return KERN_SUCCESS;
}
kern_return_t
pmap_unnest_options(
pmap_t grand,
addr64_t vaddr,
uint64_t size,
unsigned int option)
{
kern_return_t kr = KERN_FAILURE;
PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr));
kr = pmap_unnest_options_internal(grand, vaddr, size, option);
PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, kr);
return kr;
}
boolean_t
pmap_adjust_unnest_parameters(
__unused pmap_t p,
__unused vm_map_offset_t *s,
__unused vm_map_offset_t *e)
{
return TRUE;
}
#if DEVELOPMENT || DEBUG
void
pmap_disable_NX(
pmap_t pmap)
{
pmap->nx_enabled = FALSE;
}
#else
void
pmap_disable_NX(
__unused pmap_t pmap)
{
}
#endif
void
pt_fake_zone_init(
int zone_index)
{
pt_fake_zone_index = zone_index;
}
void
pt_fake_zone_info(
int *count,
vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
{
*count = inuse_pmap_pages_count;
*cur_size = PAGE_SIZE * (inuse_pmap_pages_count);
*max_size = PAGE_SIZE * (inuse_pmap_pages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count);
*elem_size = PAGE_SIZE;
*alloc_size = PAGE_SIZE;
*sum_size = (alloc_pmap_pages_count) * PAGE_SIZE;
*collectable = 1;
*exhaustable = 0;
*caller_acct = 1;
}
#define ARM_FULL_TLB_FLUSH_THRESHOLD 64
#define ARM64_FULL_TLB_FLUSH_THRESHOLD 256
static void
flush_mmu_tlb_region_asid(
vm_offset_t va,
unsigned length,
pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
vm_offset_t end = va + length;
uint32_t asid;
asid = pmap->asid;
if (length / ARM_SMALL_PAGE_SIZE > ARM_FULL_TLB_FLUSH_THRESHOLD) {
boolean_t flush_all = FALSE;
if ((asid == 0) || (pmap->nested == TRUE))
flush_all = TRUE;
if (flush_all)
flush_mmu_tlb();
else
flush_mmu_tlb_asid(asid);
return;
}
if (pmap->nested == TRUE) {
#if !__ARM_MP_EXT__
flush_mmu_tlb();
#else
va = arm_trunc_page(va);
while (va < end) {
flush_mmu_tlb_mva_entries(va);
va += ARM_SMALL_PAGE_SIZE;
}
#endif
return;
}
va = arm_trunc_page(va) | (asid & 0xff);
flush_mmu_tlb_entries(va, end);
#else
vm_offset_t end = va + length;
uint32_t asid;
asid = pmap->asid;
if ((length >> ARM_TT_L3_SHIFT) > ARM64_FULL_TLB_FLUSH_THRESHOLD) {
boolean_t flush_all = FALSE;
if ((asid == 0) || (pmap->nested == TRUE))
flush_all = TRUE;
if (flush_all)
flush_mmu_tlb();
else
flush_mmu_tlb_asid((uint64_t)asid << TLBI_ASID_SHIFT);
return;
}
va = tlbi_asid(asid) | tlbi_addr(va);
end = tlbi_asid(asid) | tlbi_addr(end);
if (pmap->nested == TRUE) {
flush_mmu_tlb_allentries(va, end);
} else {
flush_mmu_tlb_entries(va, end);
}
#endif
}
void
flush_mmu_tlb_region(
vm_offset_t va,
unsigned length)
{
flush_mmu_tlb_region_asid(va, length, kernel_pmap);
}
unsigned int
pmap_cache_attributes(
ppnum_t pn)
{
pmap_paddr_t paddr;
int pai;
unsigned int result;
pp_attr_t pp_attr_current;
paddr = ptoa(pn);
if ((paddr >= io_rgn_start) && (paddr < io_rgn_end)) {
unsigned int attr = IO_ATTR_WIMG(io_attr_table[(paddr - io_rgn_start) / io_rgn_granule]);
if (attr)
return attr;
else
return (VM_WIMG_IO);
}
if (!pmap_initialized) {
if ((paddr >= gPhysBase) && (paddr < gPhysBase+gPhysSize))
return (VM_WIMG_DEFAULT);
else
return (VM_WIMG_IO);
}
if (!pa_valid(paddr))
return (VM_WIMG_IO);
result = VM_WIMG_DEFAULT;
pai = (int)pa_index(paddr);
pp_attr_current = pp_attr_table[pai];
if (pp_attr_current & PP_ATTR_WIMG_MASK)
result = pp_attr_current & PP_ATTR_WIMG_MASK;
return result;
}
static boolean_t
pmap_batch_set_cache_attributes_internal(
ppnum_t pn,
unsigned int cacheattr,
unsigned int page_cnt,
unsigned int page_index,
boolean_t doit,
unsigned int *res)
{
pmap_paddr_t paddr;
int pai;
pp_attr_t pp_attr_current;
pp_attr_t pp_attr_template;
unsigned int wimg_bits_prev, wimg_bits_new;
if (cacheattr & VM_WIMG_USE_DEFAULT)
cacheattr = VM_WIMG_DEFAULT;
if ((doit == FALSE) && (*res == 0)) {
*res = page_cnt;
if (platform_cache_batch_wimg(cacheattr & (VM_WIMG_MASK), page_cnt<<PAGE_SHIFT) == FALSE) {
return FALSE;
}
}
paddr = ptoa(pn);
if (!pa_valid(paddr)) {
panic("pmap_batch_set_cache_attributes(): pn 0x%08x not managed\n", pn);
}
pai = (int)pa_index(paddr);
if (doit)
LOCK_PVH(pai);
pp_attr_current = pp_attr_table[pai];
wimg_bits_prev = VM_WIMG_DEFAULT;
if (pp_attr_current & PP_ATTR_WIMG_MASK)
wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK;
pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK));
if (doit)
pp_attr_table[pai] = pp_attr_template;
wimg_bits_new = VM_WIMG_DEFAULT;
if (pp_attr_template & PP_ATTR_WIMG_MASK)
wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK;
if (doit) {
if (wimg_bits_new != wimg_bits_prev)
pmap_update_cache_attributes_locked(pn, cacheattr);
UNLOCK_PVH(pai);
} else {
if (wimg_bits_new == VM_WIMG_COPYBACK) {
return FALSE;
}
if (wimg_bits_prev == wimg_bits_new) {
*res = *res-1;
if (!platform_cache_batch_wimg(wimg_bits_new, (*res)<<PAGE_SHIFT)) {
return FALSE;
}
}
return TRUE;
}
if (page_cnt == (page_index+1)) {
wimg_bits_prev = VM_WIMG_COPYBACK;
if (((page_cnt == (page_index+1)) && (wimg_bits_prev != wimg_bits_new))
&& ((wimg_bits_prev == VM_WIMG_COPYBACK)
|| ((wimg_bits_prev == VM_WIMG_INNERWBACK)
&& (wimg_bits_new != VM_WIMG_COPYBACK))
|| ((wimg_bits_prev == VM_WIMG_WTHRU)
&& ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK))))) {
platform_cache_flush_wimg(wimg_bits_new);
}
}
return TRUE;
};
boolean_t
pmap_batch_set_cache_attributes(
ppnum_t pn,
unsigned int cacheattr,
unsigned int page_cnt,
unsigned int page_index,
boolean_t doit,
unsigned int *res)
{
return pmap_batch_set_cache_attributes_internal(pn, cacheattr, page_cnt, page_index, doit, res);
}
static void
pmap_set_cache_attributes_internal(
ppnum_t pn,
unsigned int cacheattr)
{
pmap_paddr_t paddr;
int pai;
pp_attr_t pp_attr_current;
pp_attr_t pp_attr_template;
unsigned int wimg_bits_prev, wimg_bits_new;
paddr = ptoa(pn);
if (!pa_valid(paddr)) {
return;
}
if (cacheattr & VM_WIMG_USE_DEFAULT)
cacheattr = VM_WIMG_DEFAULT;
pai = (int)pa_index(paddr);
LOCK_PVH(pai);
pp_attr_current = pp_attr_table[pai];
wimg_bits_prev = VM_WIMG_DEFAULT;
if (pp_attr_current & PP_ATTR_WIMG_MASK)
wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK;
pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK)) ;
pp_attr_table[pai] = pp_attr_template;
wimg_bits_new = VM_WIMG_DEFAULT;
if (pp_attr_template & PP_ATTR_WIMG_MASK)
wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK;
if (wimg_bits_new != wimg_bits_prev)
pmap_update_cache_attributes_locked(pn, cacheattr);
UNLOCK_PVH(pai);
if ((wimg_bits_prev != wimg_bits_new)
&& ((wimg_bits_prev == VM_WIMG_COPYBACK)
|| ((wimg_bits_prev == VM_WIMG_INNERWBACK)
&& (wimg_bits_new != VM_WIMG_COPYBACK))
|| ((wimg_bits_prev == VM_WIMG_WTHRU)
&& ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK)))))
pmap_sync_page_attributes_phys(pn);
}
void
pmap_set_cache_attributes(
ppnum_t pn,
unsigned int cacheattr)
{
pmap_set_cache_attributes_internal(pn, cacheattr);
}
void
pmap_update_cache_attributes_locked(
ppnum_t ppnum,
unsigned attributes)
{
pmap_paddr_t phys = ptoa(ppnum);
pv_entry_t *pve_p;
pt_entry_t *pte_p;
pv_entry_t **pv_h;
pt_entry_t tmplate;
unsigned int pai;
#if (__ARM_VMSA__ == 7)
#define ARM_PTE_SHMASK ARM_PTE_SH
#endif
#if __ARM_PTE_PHYSMAP__
vm_offset_t kva = phystokv(phys);
pte_p = pmap_pte(kernel_pmap, kva);
tmplate = *pte_p;
tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);
tmplate |= wimg_to_pte(attributes);
WRITE_PTE(pte_p, tmplate);
PMAP_UPDATE_TLBS(kernel_pmap, kva, kva + PAGE_SIZE);
#endif
pai = (unsigned int)pa_index(phys);
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
pte_p = PT_ENTRY_NULL;
}
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pmap_t pmap;
if (pve_p != PV_ENTRY_NULL)
pte_p = pve_get_ptep(pve_p);
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
tmplate = *pte_p;
tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);
tmplate |= wimg_to_pte(attributes);
WRITE_PTE(pte_p, tmplate);
PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
pte_p = PT_ENTRY_NULL;
if (pve_p != PV_ENTRY_NULL)
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
#if (__ARM_VMSA__ == 7)
vm_map_address_t
pmap_create_sharedpage(
void)
{
pmap_paddr_t pa;
kern_return_t kr;
(void) pmap_pages_alloc(&pa, PAGE_SIZE, 0);
memset((char *) phystokv(pa), 0, PAGE_SIZE);
kr = pmap_enter(kernel_pmap, _COMM_PAGE_BASE_ADDRESS, atop(pa), VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
return((vm_map_address_t)phystokv(pa));
}
#else
static void
pmap_update_tt3e(
pmap_t pmap,
vm_address_t address,
tt_entry_t template)
{
tt_entry_t *ptep, pte;
ptep = pmap_tt3e(pmap, address);
if (ptep == NULL) {
panic("%s: no ptep?\n", __FUNCTION__);
}
pte = *ptep;
pte = tte_to_pa(pte) | template;
WRITE_PTE(ptep, pte);
}
#define PMAP_COMM_PAGE_PTE_TEMPLATE (ARM_PTE_TYPE_VALID \
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \
| ARM_PTE_SH(SH_INNER_MEMORY) | ARM_PTE_NX \
| ARM_PTE_PNX | ARM_PTE_AP(AP_RORO) | ARM_PTE_AF)
vm_map_address_t
pmap_create_sharedpage(
void
)
{
kern_return_t kr;
pmap_paddr_t pa = 0;
(void) pmap_pages_alloc(&pa, PAGE_SIZE, 0);
memset((char *) phystokv(pa), 0, PAGE_SIZE);
kr = pmap_expand(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL);
assert(kr == KERN_SUCCESS);
kr = pmap_enter(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE | ARM_PTE_NG);
kr = pmap_expand(kernel_pmap, _COMM_PRIV_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL);
assert(kr == KERN_SUCCESS);
kr = pmap_enter(kernel_pmap, _COMM_PRIV_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
sharedpage_pmap = pmap_create(NULL, 0x0, FALSE);
assert(sharedpage_pmap != NULL);
kr = pmap_enter(sharedpage_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
kr = pmap_enter(sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
sharedpage_rw_addr = phystokv(pa);
return((vm_map_address_t)sharedpage_rw_addr);
}
#if (ARM_PGSHIFT == 14) || __ARM64_TWO_LEVEL_PMAP__
static_assert((_COMM_PAGE64_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= MACH_VM_MAX_ADDRESS);
static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= VM_MAX_ADDRESS);
#elif (ARM_PGSHIFT == 12)
static_assert((_COMM_PAGE64_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= MACH_VM_MAX_ADDRESS);
static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= VM_MAX_ADDRESS);
#else
#error Nested shared page mapping is unsupported on this config
#endif
static void
pmap_insert_sharedpage_internal(
pmap_t pmap)
{
#if (ARM_PGSHIFT == 14) && !__ARM64_TWO_LEVEL_PMAP__
kern_return_t kr;
#endif
vm_offset_t sharedpage_vaddr;
pt_entry_t *ttep, *src_ttep;
#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
#error We assume a single page.
#endif
if (pmap_is_64bit(pmap)) {
sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS;
} else {
sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS;
}
PMAP_LOCK(pmap);
#if (ARM_PGSHIFT == 12)
#if __ARM64_TWO_LEVEL_PMAP__
#error A two level page table with a page shift of 12 is not currently supported
#endif
ttep = pmap_tt1e(pmap, sharedpage_vaddr);
if (*ttep != ARM_PTE_EMPTY) {
panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
}
src_ttep = pmap_tt1e(sharedpage_pmap, sharedpage_vaddr);
#elif (ARM_PGSHIFT == 14)
#if !__ARM64_TWO_LEVEL_PMAP__
while (*pmap_tt1e(pmap, sharedpage_vaddr) == ARM_PTE_EMPTY) {
PMAP_UNLOCK(pmap);
kr = pmap_expand(pmap, _COMM_PAGE32_BASE_ADDRESS, 0, PMAP_TT_L2_LEVEL);
if (kr != KERN_SUCCESS) {
panic("Failed to pmap_expand for 32-bit commpage, pmap=%p", pmap);
}
PMAP_LOCK(pmap);
}
#endif
ttep = pmap_tt2e(pmap, sharedpage_vaddr);
if (*ttep != ARM_PTE_EMPTY) {
panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
}
src_ttep = pmap_tt2e(sharedpage_pmap, sharedpage_vaddr);
#endif
*ttep = *src_ttep;
#ifndef __ARM_L1_PTW__
CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t));
#endif
flush_mmu_tlb_region(sharedpage_vaddr, PAGE_SIZE);
#if (ARM_PGSHIFT == 12) && !__ARM64_TWO_LEVEL_PMAP__
flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));
#elif (ARM_PGSHIFT == 14)
flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
#endif
PMAP_UNLOCK(pmap);
}
static void
pmap_sharedpage_flush_32_to_64(
void)
{
flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE);
}
static void
pmap_unmap_sharedpage(
pmap_t pmap)
{
pt_entry_t *ttep;
vm_offset_t sharedpage_vaddr;
#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
#error We assume a single page.
#endif
if (pmap_is_64bit(pmap)) {
sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS;
} else {
sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS;
}
#if (ARM_PGSHIFT == 12)
#if __ARM64_TWO_LEVEL_PMAP__
#error A two level page table with a page shift of 12 is not currently supported
#endif
ttep = pmap_tt1e(pmap, sharedpage_vaddr);
if (ttep == NULL) {
return;
}
if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(sharedpage_pmap, sharedpage_vaddr)) {
panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
}
#elif (ARM_PGSHIFT == 14)
ttep = pmap_tt2e(pmap, sharedpage_vaddr);
if (ttep == NULL) {
return;
}
if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(sharedpage_pmap, sharedpage_vaddr)) {
panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
}
#endif
*ttep = ARM_TTE_EMPTY;
flush_mmu_tlb_region(sharedpage_vaddr, PAGE_SIZE);
#if (ARM_PGSHIFT == 12)
#if __ARM64_TWO_LEVEL_PMAP__
#error A two level page table with a page shift of 12 is not currently supported
#endif
flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));
#elif (ARM_PGSHIFT == 14)
flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
#endif
}
void
pmap_insert_sharedpage(
pmap_t pmap)
{
pmap_insert_sharedpage_internal(pmap);
}
static boolean_t
pmap_is_64bit(
pmap_t pmap)
{
return (pmap->is_64bit);
}
#endif
boolean_t
pmap_valid_page(
ppnum_t pn) {
return pa_valid(ptoa(pn));
}
static boolean_t
pmap_is_empty_internal(
pmap_t pmap,
vm_map_offset_t va_start,
vm_map_offset_t va_end)
{
vm_map_offset_t block_start, block_end;
tt_entry_t *tte_p;
if (pmap == NULL) {
return TRUE;
}
if ((pmap != kernel_pmap) && (not_in_kdp)) {
PMAP_LOCK(pmap);
}
#if (__ARM_VMSA__ == 7)
if (tte_index(pmap, va_end) >= pmap->tte_index_max) {
if ((pmap != kernel_pmap) && (not_in_kdp)) {
PMAP_UNLOCK(pmap);
}
return TRUE;
}
block_start = va_start;
tte_p = pmap_tte(pmap, block_start);
while (block_start < va_end) {
block_end = (block_start + ARM_TT_L1_SIZE) & ~(ARM_TT_L1_OFFMASK);
if (block_end > va_end)
block_end = va_end;
if ((*tte_p & ARM_TTE_TYPE_MASK) != 0) {
vm_map_offset_t offset;
ppnum_t phys_page = 0;
for (offset = block_start;
offset < block_end;
offset += ARM_PGBYTES) {
phys_page = pmap_vtophys(pmap, offset);
if (phys_page) {
if ((pmap != kernel_pmap) && (not_in_kdp)) {
PMAP_UNLOCK(pmap);
}
return FALSE;
}
}
}
block_start = block_end;
tte_p++;
}
#else
block_start = va_start;
while (block_start < va_end) {
pt_entry_t *bpte_p, *epte_p;
pt_entry_t *pte_p;
block_end = (block_start + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK;
if (block_end > va_end)
block_end = va_end;
tte_p = pmap_tt2e(pmap, block_start);
if ((tte_p != PT_ENTRY_NULL)
&& ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte_p = &pte_p[tt3_index(pmap, block_start)];
epte_p = bpte_p + (((block_end - block_start) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT);
for (pte_p = bpte_p; pte_p < epte_p; pte_p++) {
if (*pte_p != ARM_PTE_EMPTY) {
if ((pmap != kernel_pmap) && (not_in_kdp)) {
PMAP_UNLOCK(pmap);
}
return FALSE;
}
}
}
block_start = block_end;
}
#endif
if ((pmap != kernel_pmap) && (not_in_kdp)) {
PMAP_UNLOCK(pmap);
}
return TRUE;
}
boolean_t
pmap_is_empty(
pmap_t pmap,
vm_map_offset_t va_start,
vm_map_offset_t va_end)
{
return pmap_is_empty_internal(pmap, va_start, va_end);
}
vm_map_offset_t pmap_max_offset(
boolean_t is64 __unused,
unsigned int option)
{
vm_map_offset_t max_offset_ret = 0;
#if defined(__arm64__)
assert (is64);
vm_map_offset_t min_max_offset = SHARED_REGION_BASE_ARM64 + SHARED_REGION_SIZE_ARM64 + 0x20000000; if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) {
max_offset_ret = arm64_pmap_max_offset_default;
} else if (option == ARM_PMAP_MAX_OFFSET_MIN) {
max_offset_ret = min_max_offset;
} else if (option == ARM_PMAP_MAX_OFFSET_MAX) {
max_offset_ret = MACH_VM_MAX_ADDRESS;
} else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) {
if (arm64_pmap_max_offset_default) {
max_offset_ret = arm64_pmap_max_offset_default;
} else if (max_mem > 0xC0000000) {
max_offset_ret = 0x0000000318000000ULL; } else if (max_mem > 0x40000000) {
max_offset_ret = 0x0000000218000000ULL; } else {
max_offset_ret = min_max_offset;
}
} else if (option == ARM_PMAP_MAX_OFFSET_JUMBO) {
max_offset_ret = 0x0000000518000000ULL; } else {
panic("pmap_max_offset illegal option 0x%x\n", option);
}
assert(max_offset_ret >= min_max_offset);
return max_offset_ret;
#else
if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) {
max_offset_ret = arm_pmap_max_offset_default;
} else if (option == ARM_PMAP_MAX_OFFSET_MIN) {
max_offset_ret = 0x66000000;
} else if (option == ARM_PMAP_MAX_OFFSET_MAX) {
max_offset_ret = VM_MAX_ADDRESS;
} else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) {
if (arm_pmap_max_offset_default) {
max_offset_ret = arm_pmap_max_offset_default;
} else if (max_mem > 0x20000000) {
max_offset_ret = 0x80000000;
} else {
max_offset_ret = 0x66000000;
}
} else {
panic("pmap_max_offset illegal option 0x%x\n", option);
}
return max_offset_ret;
#endif
}
#if CONFIG_DTRACE
extern kern_return_t dtrace_copyio_preflight(addr64_t);
extern kern_return_t dtrace_copyio_postflight(addr64_t);
kern_return_t dtrace_copyio_preflight(
__unused addr64_t va)
{
if (current_map() == kernel_map)
return KERN_FAILURE;
else
return KERN_SUCCESS;
}
kern_return_t dtrace_copyio_postflight(
__unused addr64_t va)
{
return KERN_SUCCESS;
}
#endif
void
pmap_flush_context_init(__unused pmap_flush_context *pfc)
{
}
void
pmap_flush(
__unused pmap_flush_context *cpus_to_flush)
{
return;
}
static boolean_t
pmap_query_resident_internal(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
mach_vm_size_t *resident_bytes_p,
mach_vm_size_t *compressed_bytes_p)
{
mach_vm_size_t resident_bytes = 0;
mach_vm_size_t compressed_bytes = 0;
pt_entry_t *bpte, *epte;
pt_entry_t *pte_p;
tt_entry_t *tte_p;
if (pmap == NULL) {
return FALSE;
}
assert(!(start % ARM_PGBYTES));
assert(!(end % ARM_PGBYTES));
assert(end >= start);
assert((end - start) <= (PTE_PGENTRIES * ARM_PGBYTES));
PMAP_LOCK(pmap);
tte_p = pmap_tte(pmap, start);
if (tte_p == (tt_entry_t *) NULL) {
PMAP_UNLOCK(pmap);
return FALSE;
}
if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
#if (__ARM_VMSA__ == 7)
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[ptenum(start)];
epte = bpte + atop(end - start);
#else
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[tt3_index(pmap, start)];
epte = bpte + ((end - start) >> ARM_TT_L3_SHIFT);
#endif
for (; bpte < epte; bpte++) {
if (ARM_PTE_IS_COMPRESSED(*bpte)) {
compressed_bytes += ARM_PGBYTES;
} else if (pa_valid(pte_to_pa(*bpte))) {
resident_bytes += ARM_PGBYTES;
}
}
}
PMAP_UNLOCK(pmap);
if (compressed_bytes_p) {
*compressed_bytes_p += compressed_bytes;
}
if (resident_bytes_p) {
*resident_bytes_p += resident_bytes;
}
return TRUE;
}
mach_vm_size_t
pmap_query_resident(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
mach_vm_size_t *compressed_bytes_p)
{
mach_vm_size_t resident_bytes;
mach_vm_size_t compressed_bytes;
vm_map_address_t va;
if (pmap == PMAP_NULL) {
if (compressed_bytes_p) {
*compressed_bytes_p = 0;
}
return 0;
}
resident_bytes = 0;
compressed_bytes = 0;
PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start),
VM_KERNEL_ADDRHIDE(end));
va = start;
while (va < end) {
vm_map_address_t l;
l = ((va + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK);
if (l > end)
l = end;
if (!pmap_query_resident_internal(pmap, va, l, &resident_bytes, compressed_bytes_p)) {
break;
}
va = l;
}
if (compressed_bytes_p) {
*compressed_bytes_p = compressed_bytes;
}
PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,
resident_bytes);
return resident_bytes;
}
#if MACH_ASSERT
extern int pmap_ledgers_panic;
static void
pmap_check_ledgers(
pmap_t pmap)
{
ledger_amount_t bal;
int pid;
char *procname;
boolean_t do_panic;
if (pmap->pmap_pid == 0) {
return;
}
do_panic = FALSE;
pid = pmap->pmap_pid;
procname = pmap->pmap_procname;
pmap_ledgers_drift.num_pmaps_checked++;
ledger_get_balance(pmap->ledger,
task_ledgers.phys_footprint,
&bal);
if (bal != 0) {
#if DEVELOPMENT || DEBUG
#endif
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"phys_footprint\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.phys_footprint_over++;
pmap_ledgers_drift.phys_footprint_over_total += bal;
if (bal > pmap_ledgers_drift.phys_footprint_over_max) {
pmap_ledgers_drift.phys_footprint_over_max = bal;
}
} else {
pmap_ledgers_drift.phys_footprint_under++;
pmap_ledgers_drift.phys_footprint_under_total += bal;
if (bal < pmap_ledgers_drift.phys_footprint_under_max) {
pmap_ledgers_drift.phys_footprint_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.internal,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"internal\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.internal_over++;
pmap_ledgers_drift.internal_over_total += bal;
if (bal > pmap_ledgers_drift.internal_over_max) {
pmap_ledgers_drift.internal_over_max = bal;
}
} else {
pmap_ledgers_drift.internal_under++;
pmap_ledgers_drift.internal_under_total += bal;
if (bal < pmap_ledgers_drift.internal_under_max) {
pmap_ledgers_drift.internal_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.internal_compressed,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"internal_compressed\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.internal_compressed_over++;
pmap_ledgers_drift.internal_compressed_over_total += bal;
if (bal > pmap_ledgers_drift.internal_compressed_over_max) {
pmap_ledgers_drift.internal_compressed_over_max = bal;
}
} else {
pmap_ledgers_drift.internal_compressed_under++;
pmap_ledgers_drift.internal_compressed_under_total += bal;
if (bal < pmap_ledgers_drift.internal_compressed_under_max) {
pmap_ledgers_drift.internal_compressed_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.iokit_mapped,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"iokit_mapped\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.iokit_mapped_over++;
pmap_ledgers_drift.iokit_mapped_over_total += bal;
if (bal > pmap_ledgers_drift.iokit_mapped_over_max) {
pmap_ledgers_drift.iokit_mapped_over_max = bal;
}
} else {
pmap_ledgers_drift.iokit_mapped_under++;
pmap_ledgers_drift.iokit_mapped_under_total += bal;
if (bal < pmap_ledgers_drift.iokit_mapped_under_max) {
pmap_ledgers_drift.iokit_mapped_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.alternate_accounting,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"alternate_accounting\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.alternate_accounting_over++;
pmap_ledgers_drift.alternate_accounting_over_total += bal;
if (bal > pmap_ledgers_drift.alternate_accounting_over_max) {
pmap_ledgers_drift.alternate_accounting_over_max = bal;
}
} else {
pmap_ledgers_drift.alternate_accounting_under++;
pmap_ledgers_drift.alternate_accounting_under_total += bal;
if (bal < pmap_ledgers_drift.alternate_accounting_under_max) {
pmap_ledgers_drift.alternate_accounting_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.alternate_accounting_compressed,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"alternate_accounting_compressed\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.alternate_accounting_compressed_over++;
pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal;
if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) {
pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal;
}
} else {
pmap_ledgers_drift.alternate_accounting_compressed_under++;
pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal;
if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) {
pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.page_table,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"page_table\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.page_table_over++;
pmap_ledgers_drift.page_table_over_total += bal;
if (bal > pmap_ledgers_drift.page_table_over_max) {
pmap_ledgers_drift.page_table_over_max = bal;
}
} else {
pmap_ledgers_drift.page_table_under++;
pmap_ledgers_drift.page_table_under_total += bal;
if (bal < pmap_ledgers_drift.page_table_under_max) {
pmap_ledgers_drift.page_table_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.purgeable_volatile,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"purgeable_volatile\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.purgeable_volatile_over++;
pmap_ledgers_drift.purgeable_volatile_over_total += bal;
if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) {
pmap_ledgers_drift.purgeable_volatile_over_max = bal;
}
} else {
pmap_ledgers_drift.purgeable_volatile_under++;
pmap_ledgers_drift.purgeable_volatile_under_total += bal;
if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) {
pmap_ledgers_drift.purgeable_volatile_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.purgeable_nonvolatile,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"purgeable_nonvolatile\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.purgeable_nonvolatile_over++;
pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal;
if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) {
pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal;
}
} else {
pmap_ledgers_drift.purgeable_nonvolatile_under++;
pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal;
if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) {
pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.purgeable_volatile_compressed,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"purgeable_volatile_compressed\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.purgeable_volatile_compressed_over++;
pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal;
if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) {
pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal;
}
} else {
pmap_ledgers_drift.purgeable_volatile_compressed_under++;
pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal;
if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) {
pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal;
}
}
}
ledger_get_balance(pmap->ledger,
task_ledgers.purgeable_nonvolatile_compressed,
&bal);
if (bal != 0) {
do_panic = TRUE;
printf("LEDGER BALANCE proc %d (%s) "
"\"purgeable_nonvolatile_compressed\" = %lld\n",
pid, procname, bal);
if (bal > 0) {
pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++;
pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal;
if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) {
pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal;
}
} else {
pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++;
pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal;
if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) {
pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal;
}
}
}
if (do_panic) {
if (pmap_ledgers_panic) {
panic("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
pmap, pid, procname);
} else {
printf("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n",
pmap, pid, procname);
}
}
PMAP_STATS_ASSERTF(pmap->stats.resident_count == 0, pmap, "stats.resident_count %d", pmap->stats.resident_count);
#if 00
PMAP_STATS_ASSERTF(pmap->stats.wired_count == 0, pmap, "stats.wired_count %d", pmap->stats.wired_count);
#endif
PMAP_STATS_ASSERTF(pmap->stats.device == 0, pmap, "stats.device %d", pmap->stats.device);
PMAP_STATS_ASSERTF(pmap->stats.internal == 0, pmap, "stats.internal %d", pmap->stats.internal);
PMAP_STATS_ASSERTF(pmap->stats.external == 0, pmap, "stats.external %d", pmap->stats.external);
PMAP_STATS_ASSERTF(pmap->stats.reusable == 0, pmap, "stats.reusable %d", pmap->stats.reusable);
PMAP_STATS_ASSERTF(pmap->stats.compressed == 0, pmap, "stats.compressed %lld", pmap->stats.compressed);
}
#endif
void pmap_advise_pagezero_range(__unused pmap_t p, __unused uint64_t a) {
}
#if CONFIG_PGTRACE
#define PROF_START uint64_t t, nanot;\
t = mach_absolute_time();
#define PROF_END absolutetime_to_nanoseconds(mach_absolute_time()-t, &nanot);\
kprintf("%s: took %llu ns\n", __func__, nanot);
#define PMAP_PGTRACE_LOCK(p) \
do { \
*(p) = ml_set_interrupts_enabled(false); \
if (simple_lock_try(&(pmap_pgtrace.lock))) break; \
ml_set_interrupts_enabled(*(p)); \
} while (true)
#define PMAP_PGTRACE_UNLOCK(p) \
do { \
simple_unlock(&(pmap_pgtrace.lock)); \
ml_set_interrupts_enabled(*(p)); \
} while (0)
#define PGTRACE_WRITE_PTE(pte_p, pte_entry) \
do { \
*(pte_p) = (pte_entry); \
FLUSH_PTE(pte_p); \
} while (0)
#define PGTRACE_MAX_MAP 16 // maximum supported va to same pa
typedef enum {
UNDEFINED,
PA_UNDEFINED,
VA_UNDEFINED,
DEFINED
} pmap_pgtrace_page_state_t;
typedef struct {
queue_chain_t chain;
pmap_paddr_t pa;
queue_head_t maps;
queue_head_t map_pool;
queue_head_t map_waste;
pmap_pgtrace_page_state_t state;
} pmap_pgtrace_page_t;
static struct {
queue_head_t pages;
decl_simple_lock_data(, lock);
} pmap_pgtrace = {};
static void pmap_pgtrace_init(void)
{
queue_init(&(pmap_pgtrace.pages));
simple_lock_init(&(pmap_pgtrace.lock), 0);
boolean_t enabled;
if (PE_parse_boot_argn("pgtrace", &enabled, sizeof(enabled))) {
pgtrace_enabled = enabled;
}
}
inline static pmap_pgtrace_page_t *pmap_pgtrace_find_page(pmap_paddr_t pa)
{
queue_head_t *q = &(pmap_pgtrace.pages);
pmap_pgtrace_page_t *p;
queue_iterate(q, p, pmap_pgtrace_page_t *, chain) {
if (p->state == UNDEFINED) {
continue;
}
if (p->state == PA_UNDEFINED) {
continue;
}
if (p->pa == pa) {
return p;
}
}
return NULL;
}
static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end)
{
bool ints;
queue_head_t *q = &(pmap_pgtrace.pages);
pmap_paddr_t pa_page;
pt_entry_t *ptep, *cptep;
pmap_pgtrace_page_t *p;
bool found = false;
PMAP_ASSERT_LOCKED(pmap);
assert(va_page == arm_trunc_page(va_page));
PMAP_PGTRACE_LOCK(&ints);
ptep = pmap_pte(pmap, va_page);
if (!ptep || !(*ptep & ARM_PTE_TYPE_VALID)) {
PMAP_PGTRACE_UNLOCK(&ints);
return false;
}
queue_head_t *mapq;
queue_head_t *mappool;
pmap_pgtrace_map_t *map = NULL;
pa_page = pte_to_pa(*ptep);
queue_iterate(q, p, pmap_pgtrace_page_t *, chain) {
mapq = &(p->maps);
mappool = &(p->map_pool);
switch (p->state) {
case PA_UNDEFINED:
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == false && map->pmap == pmap && map->ova == va_page) {
p->pa = pa_page;
map->range.start = start;
map->range.end = end;
found = true;
break;
}
}
break;
case VA_UNDEFINED:
if (p->pa != pa_page) {
break;
}
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == false) {
map->pmap = pmap;
map->ova = va_page;
map->range.start = start;
map->range.end = end;
found = true;
break;
}
}
break;
case DEFINED:
if (p->pa != pa_page) {
break;
}
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == true && map->pmap == pmap && map->ova == va_page) {
kprintf("%s: skip existing mapping at va=%llx\n", __func__, va_page);
break;
} else if (map->cloned == true && map->pmap == kernel_pmap && map->cva[1] == va_page) {
kprintf("%s: skip clone mapping at va=%llx\n", __func__, va_page);
break;
} else if (map->cloned == false && map->pmap == pmap && map->ova == va_page) {
found = true;
break;
}
}
break;
default:
panic("invalid state p->state=%x\n", p->state);
}
if (found == true) {
break;
}
}
if (found == false) {
PMAP_PGTRACE_UNLOCK(&ints);
return false;
}
for (int i = 0; i < 3; i++) {
ptep = pmap_pte(pmap, va_page + (i-1)*ARM_PGBYTES);
cptep = pmap_pte(kernel_pmap, map->cva[i]);
assert(cptep != NULL);
if (ptep == NULL) {
PGTRACE_WRITE_PTE(cptep, (pt_entry_t)NULL);
} else {
PGTRACE_WRITE_PTE(cptep, *ptep);
}
PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES);
}
ptep = pmap_pte(pmap, va_page);
cptep = pmap_pte(kernel_pmap, map->cva[1]);
PGTRACE_WRITE_PTE(ptep, (*ptep | ARM_PTE_PGTRACE) & ~ARM_PTE_TYPE_VALID);
PMAP_UPDATE_TLBS(pmap, map->ova, map->ova+ARM_PGBYTES);
map->cloned = true;
p->state = DEFINED;
kprintf("%s: pa_page=%llx va_page=%llx cva[1]=%llx pmap=%p ptep=%p cptep=%p\n", __func__, pa_page, va_page, map->cva[1], pmap, ptep, cptep);
PMAP_PGTRACE_UNLOCK(&ints);
return true;
}
static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t va)
{
bool ints, found = false;
pmap_pgtrace_page_t *p;
pt_entry_t *ptep;
PMAP_PGTRACE_LOCK(&ints);
p = pmap_pgtrace_find_page(pa);
if (p == NULL) {
goto unlock_exit;
}
queue_head_t *mapq = &(p->maps);
queue_head_t *mappool = &(p->map_pool);
pmap_pgtrace_map_t *map;
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->pmap == pmap && map->ova == va) {
found = true;
break;
}
}
if (!found) {
goto unlock_exit;
}
if (map->cloned == true) {
ptep = pmap_pte(pmap, map->ova);
assert(ptep);
PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID);
PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES);
for (int i = 0; i < 3; i++) {
ptep = pmap_pte(kernel_pmap, map->cva[i]);
assert(ptep != NULL);
PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]);
PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES);
}
}
queue_remove(mapq, map, pmap_pgtrace_map_t *, chain);
map->pmap = NULL;
map->ova = (vm_map_offset_t)NULL;
map->cloned = false;
queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain);
kprintf("%s: p=%p pa=%llx va=%llx\n", __func__, p, pa, va);
unlock_exit:
PMAP_PGTRACE_UNLOCK(&ints);
}
static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa)
{
bool ints;
pmap_pgtrace_page_t *p;
pt_entry_t *ptep;
PMAP_PGTRACE_LOCK(&ints);
p = pmap_pgtrace_find_page(pa);
if (p == NULL) {
PMAP_PGTRACE_UNLOCK(&ints);
return;
}
queue_head_t *mapq = &(p->maps);
queue_head_t *mappool = &(p->map_pool);
queue_head_t *mapwaste = &(p->map_waste);
pmap_pgtrace_map_t *map;
while (!queue_empty(mapq)) {
queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain);
queue_enter_first(mapwaste, map, pmap_pgtrace_map_t*, chain);
}
PMAP_PGTRACE_UNLOCK(&ints);
queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == true) {
PMAP_LOCK(map->pmap);
ptep = pmap_pte(map->pmap, map->ova);
assert(ptep);
PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID);
PMAP_UPDATE_TLBS(map->pmap, map->ova, map->ova+ARM_PGBYTES);
for (int i = 0; i < 3; i++) {
ptep = pmap_pte(kernel_pmap, map->cva[i]);
assert(ptep != NULL);
PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]);
PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES);
}
PMAP_UNLOCK(map->pmap);
}
map->pmap = NULL;
map->ova = (vm_map_offset_t)NULL;
map->cloned = false;
}
PMAP_PGTRACE_LOCK(&ints);
while (!queue_empty(mapwaste)) {
queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain);
queue_enter_first(mappool, map, pmap_pgtrace_map_t*, chain);
}
PMAP_PGTRACE_UNLOCK(&ints);
}
inline static void pmap_pgtrace_get_search_space(pmap_t pmap, vm_map_offset_t *startp, vm_map_offset_t *endp)
{
uint64_t tsz;
vm_map_offset_t end;
if (pmap == kernel_pmap) {
tsz = (get_tcr() >> TCR_T1SZ_SHIFT) & TCR_TSZ_MASK;
*startp = MAX(VM_MIN_KERNEL_ADDRESS, (UINT64_MAX >> (64-tsz)) << (64-tsz));
*endp = VM_MAX_KERNEL_ADDRESS;
} else {
tsz = (get_tcr() >> TCR_T0SZ_SHIFT) & TCR_TSZ_MASK;
if (tsz == 64) {
end = 0;
} else {
end = ((uint64_t)1 << (64-tsz)) - 1;
}
*startp = 0;
*endp = end;
}
assert(*endp > *startp);
return;
}
static uint64_t pmap_pgtrace_clone_from_pa(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset) {
uint64_t ret = 0;
vm_map_offset_t min, max;
vm_map_offset_t cur_page, end_page;
pt_entry_t *ptep;
tt_entry_t *ttep;
tt_entry_t tte;
pmap_pgtrace_get_search_space(pmap, &min, &max);
cur_page = arm_trunc_page(min);
end_page = arm_trunc_page(max);
while (cur_page <= end_page) {
vm_map_offset_t add = 0;
PMAP_LOCK(pmap);
if (pmap == kernel_pmap &&
((vm_kernel_base <= cur_page && cur_page < vm_kernel_top) ||
(vm_kext_base <= cur_page && cur_page < vm_kext_top))) {
add = ARM_PGBYTES;
goto unlock_continue;
}
#if __ARM64_TWO_LEVEL_PMAP__
ttep = pmap_tt2e(pmap, cur_page);
assert(ttep);
tte = *ttep;
#else
ttep = pmap_tt1e(pmap, cur_page);
assert(ttep);
tte = *ttep;
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
add = ARM_TT_L1_SIZE;
goto unlock_continue;
}
tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, cur_page)];
#endif
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
add = ARM_TT_L2_SIZE;
goto unlock_continue;
}
ptep = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, cur_page)]);
if (ptep == PT_ENTRY_NULL) {
add = ARM_TT_L3_SIZE;
goto unlock_continue;
}
if (arm_trunc_page(pa) == pte_to_pa(*ptep)) {
if (pmap_pgtrace_enter_clone(pmap, cur_page, start_offset, end_offset) == true) {
ret++;
}
}
add = ARM_PGBYTES;
unlock_continue:
PMAP_UNLOCK(pmap);
if (cur_page + add < cur_page) {
break;
}
cur_page += add;
}
return ret;
}
static uint64_t pmap_pgtrace_clone_from_pvtable(pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset)
{
uint64_t ret = 0;
unsigned long pai;
pv_entry_t **pvh;
pt_entry_t *ptep;
pmap_t pmap;
typedef struct {
queue_chain_t chain;
pmap_t pmap;
vm_map_offset_t va;
} pmap_va_t;
queue_head_t pmapvaq;
pmap_va_t *pmapva;
queue_init(&pmapvaq);
pai = pa_index(pa);
LOCK_PVH(pai);
pvh = pai_to_pvh(pai);
if (pvh_test_type(pvh, PVH_TYPE_PTEP)) {
ptep = pvh_ptep(pvh);
pmap = ptep_get_pmap(ptep);
pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t));
pmapva->pmap = pmap;
pmapva->va = ptep_get_va(ptep);
queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain);
} else if (pvh_test_type(pvh, PVH_TYPE_PVEP)) {
pv_entry_t *pvep;
pvep = pvh_list(pvh);
while (pvep) {
ptep = pve_get_ptep(pvep);
pmap = ptep_get_pmap(ptep);
pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t));
pmapva->pmap = pmap;
pmapva->va = ptep_get_va(ptep);
queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain);
pvep = PVE_NEXT_PTR(pve_next(pvep));
}
}
UNLOCK_PVH(pai);
queue_iterate(&pmapvaq, pmapva, pmap_va_t *, chain) {
PMAP_LOCK(pmapva->pmap);
ptep = pmap_pte(pmapva->pmap, pmapva->va);
if (pte_to_pa(*ptep) == pa) {
if (pmap_pgtrace_enter_clone(pmapva->pmap, pmapva->va, start_offset, end_offset) == true) {
ret++;
}
}
PMAP_UNLOCK(pmapva->pmap);
kfree(pmapva, sizeof(pmap_va_t));
}
return ret;
}
static pmap_pgtrace_page_t *pmap_pgtrace_alloc_page(void)
{
pmap_pgtrace_page_t *p;
queue_head_t *mapq;
queue_head_t *mappool;
queue_head_t *mapwaste;
pmap_pgtrace_map_t *map;
p = kalloc(sizeof(pmap_pgtrace_page_t));
assert(p);
p->state = UNDEFINED;
mapq = &(p->maps);
mappool = &(p->map_pool);
mapwaste = &(p->map_waste);
queue_init(mapq);
queue_init(mappool);
queue_init(mapwaste);
for (int i = 0; i < PGTRACE_MAX_MAP; i++) {
vm_map_offset_t newcva;
pt_entry_t *cptep;
kern_return_t kr;
vm_map_entry_t entry;
vm_object_reference(kernel_object);
kr = vm_map_find_space(kernel_map, &newcva, vm_map_round_page(3*ARM_PGBYTES, PAGE_MASK), 0, 0, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_DIAG, &entry);
if (kr != KERN_SUCCESS) {
panic("%s VM couldn't find any space kr=%d\n", __func__, kr);
}
VME_OBJECT_SET(entry, kernel_object);
VME_OFFSET_SET(entry, newcva);
vm_map_unlock(kernel_map);
map = kalloc(sizeof(pmap_pgtrace_map_t));
for (int j = 0; j < 3; j ++) {
vm_map_offset_t addr = newcva + j * ARM_PGBYTES;
kr = pmap_expand(kernel_pmap, addr, 0, PMAP_TT_MAX_LEVEL);
if (kr != KERN_SUCCESS) {
panic("%s: pmap_expand(kernel_pmap, addr=%llx) returns kr=%d\n", __func__, addr, kr);
}
cptep = pmap_pte(kernel_pmap, addr);
assert(cptep != NULL);
map->cva[j] = addr;
map->cva_spte[j] = *cptep;
}
map->range.start = map->range.end = 0;
map->cloned = false;
queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain);
}
return p;
}
static void pmap_pgtrace_free_page(pmap_pgtrace_page_t *p)
{
queue_head_t *mapq;
queue_head_t *mappool;
queue_head_t *mapwaste;
pmap_pgtrace_map_t *map;
assert(p);
mapq = &(p->maps);
mappool = &(p->map_pool);
mapwaste = &(p->map_waste);
while (!queue_empty(mapq)) {
queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain);
kfree(map, sizeof(pmap_pgtrace_map_t));
}
while (!queue_empty(mappool)) {
queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain);
kfree(map, sizeof(pmap_pgtrace_map_t));
}
while (!queue_empty(mapwaste)) {
queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain);
kfree(map, sizeof(pmap_pgtrace_map_t));
}
kfree(p, sizeof(pmap_pgtrace_page_t));
}
int pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end)
{
int ret = 0;
pt_entry_t *ptep;
queue_head_t *q = &(pmap_pgtrace.pages);
bool ints;
vm_map_offset_t cur_page, end_page;
if (start > end) {
kprintf("%s: invalid start=%llx > end=%llx\n", __func__, start, end);
return -1;
}
PROF_START
cur_page = arm_trunc_page(start);
end_page = arm_trunc_page(end);
while (cur_page <= end_page) {
pmap_paddr_t pa_page = 0;
uint64_t num_cloned = 0;
pmap_pgtrace_page_t *p = NULL, *newp;
bool free_newp = true;
pmap_pgtrace_page_state_t state;
newp = pmap_pgtrace_alloc_page();
if (pmap != NULL) {
PMAP_LOCK(pmap);
}
if (pmap != kernel_pmap) {
PMAP_LOCK(kernel_pmap);
}
if (pmap == NULL) {
ptep = NULL;
pa_page = cur_page;
state = VA_UNDEFINED;
} else {
ptep = pmap_pte(pmap, cur_page);
if (ptep != NULL) {
pa_page = pte_to_pa(*ptep);
state = DEFINED;
} else {
state = PA_UNDEFINED;
}
}
PMAP_PGTRACE_LOCK(&ints);
if (state != PA_UNDEFINED) {
p = pmap_pgtrace_find_page(pa_page);
}
if (p == NULL) {
queue_enter_first(q, newp, pmap_pgtrace_page_t *, chain);
p = newp;
free_newp = false;
}
p->state = state;
queue_head_t *mapq = &(p->maps);
queue_head_t *mappool = &(p->map_pool);
pmap_pgtrace_map_t *map;
vm_map_offset_t start_offset, end_offset;
if (cur_page > start) {
start_offset = 0;
} else {
start_offset = start-cur_page;
}
if (cur_page == end_page) {
end_offset = end-end_page;
} else {
end_offset = ARM_PGBYTES-1;
}
kprintf("%s: pmap=%p cur_page=%llx ptep=%p state=%d start_offset=%llx end_offset=%llx\n", __func__, pmap, cur_page, ptep, state, start_offset, end_offset);
assert(!queue_empty(mappool));
queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain);
if (p->state == PA_UNDEFINED) {
map->pmap = pmap;
map->ova = cur_page;
map->range.start = start_offset;
map->range.end = end_offset;
} else if (p->state == VA_UNDEFINED) {
p->pa = pa_page;
map->range.start = start_offset;
map->range.end = end_offset;
} else if (p->state == DEFINED) {
p->pa = pa_page;
map->pmap = pmap;
map->ova = cur_page;
map->range.start = start_offset;
map->range.end = end_offset;
} else {
panic("invalid p->state=%d\n", p->state);
}
map->cloned = false;
queue_enter(mapq, map, pmap_pgtrace_map_t *, chain);
PMAP_PGTRACE_UNLOCK(&ints);
if (pmap != kernel_pmap) {
PMAP_UNLOCK(kernel_pmap);
}
if (pmap != NULL) {
PMAP_UNLOCK(pmap);
}
if (pa_valid(pa_page)) {
num_cloned = pmap_pgtrace_clone_from_pvtable(pa_page, start_offset, end_offset);
}
if (pmap == NULL) {
num_cloned += pmap_pgtrace_clone_from_pa(kernel_pmap, pa_page, start_offset, end_offset);
} else {
num_cloned += pmap_pgtrace_clone_from_pa(pmap, pa_page, start_offset, end_offset);
}
if (free_newp) {
pmap_pgtrace_free_page(newp);
}
if (num_cloned == 0) {
kprintf("%s: no mapping found for pa_page=%llx but will be added when a page entered\n", __func__, pa_page);
}
ret += num_cloned;
if (cur_page + ARM_PGBYTES < cur_page) {
break;
} else {
cur_page += ARM_PGBYTES;
}
}
PROF_END
return ret;
}
int pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end)
{
int ret = 0;
bool ints;
queue_head_t *q = &(pmap_pgtrace.pages);
pmap_pgtrace_page_t *p;
vm_map_offset_t cur_page, end_page;
kprintf("%s start=%llx end=%llx\n", __func__, start, end);
PROF_START
pt_entry_t *ptep;
pmap_paddr_t pa_page;
cur_page = arm_trunc_page(start);
end_page = arm_trunc_page(end);
while (cur_page <= end_page) {
p = NULL;
if (pmap == NULL) {
pa_page = cur_page;
} else {
PMAP_LOCK(pmap);
ptep = pmap_pte(pmap, cur_page);
if (ptep == NULL) {
PMAP_UNLOCK(pmap);
goto cont;
}
pa_page = pte_to_pa(*ptep);
PMAP_UNLOCK(pmap);
}
pmap_pgtrace_remove_all_clone(pa_page);
PMAP_PGTRACE_LOCK(&ints);
p = pmap_pgtrace_find_page(pa_page);
if (p != NULL) {
queue_remove(q, p, pmap_pgtrace_page_t *, chain);
ret++;
}
PMAP_PGTRACE_UNLOCK(&ints);
if (p != NULL) {
pmap_pgtrace_free_page(p);
}
cont:
if (cur_page + ARM_PGBYTES < cur_page) {
break;
} else {
cur_page += ARM_PGBYTES;
}
}
PROF_END
return ret;
}
kern_return_t pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss)
{
pt_entry_t *ptep;
pgtrace_run_result_t res;
pmap_pgtrace_page_t *p;
bool ints, found = false;
pmap_paddr_t pa;
ptep = pmap_pte(pmap, va);
if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) {
return KERN_FAILURE;
}
PMAP_PGTRACE_LOCK(&ints);
ptep = pmap_pte(pmap, va);
if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) {
PMAP_PGTRACE_UNLOCK(&ints);
return KERN_FAILURE;
} else if ((*ptep & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE_VALID) {
kprintf("%s Somehow this cpu's tlb has not updated?\n", __func__);
PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES);
PMAP_PGTRACE_UNLOCK(&ints);
return KERN_SUCCESS;
}
pa = pte_to_pa(*ptep);
p = pmap_pgtrace_find_page(arm_trunc_page(pa));
if (p == NULL) {
panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa);
}
queue_head_t *mapq = &(p->maps);
queue_head_t *mapwaste = &(p->map_waste);
pmap_pgtrace_map_t *map;
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->pmap == pmap && map->ova == arm_trunc_page(va)) {
found = true;
break;
}
}
if (!found) {
queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) {
if (map->pmap == pmap && map->ova == arm_trunc_page(va)) {
found = true;
break;
}
}
}
if (!found) {
panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa);
}
bzero(&res, sizeof(res));
pgtrace_decode_and_run(*(uint32_t *)get_saved_state_pc(ss), va, map->cva, ss, &res);
vm_map_offset_t offset = va - map->ova;
if (map->range.start <= offset && offset <= map->range.end) {
pgtrace_write_log(res);
}
PMAP_PGTRACE_UNLOCK(&ints);
set_saved_state_pc(ss, get_saved_state_pc(ss) + sizeof(uint32_t));
return KERN_SUCCESS;
}
#endif
boolean_t
pmap_enforces_execute_only(
#if (__ARM_VMSA__ == 7)
__unused
#endif
pmap_t pmap)
{
#if (__ARM_VMSA__ > 7)
return (pmap != kernel_pmap);
#else
return FALSE;
#endif
}
void
pmap_set_jit_entitled(
__unused pmap_t pmap)
{
return;
}
static kern_return_t
pmap_query_page_info_internal(
pmap_t pmap,
vm_map_offset_t va,
int *disp_p)
{
int disp;
pmap_paddr_t pa;
int pai;
pt_entry_t *pte;
pv_entry_t **pv_h, *pve_p;
if (pmap == PMAP_NULL || pmap == kernel_pmap) {
*disp_p = 0;
return KERN_INVALID_ARGUMENT;
}
disp = 0;
PMAP_LOCK(pmap);
pte = pmap_pte(pmap, va);
if (pte == PT_ENTRY_NULL) {
goto done;
}
pa = pte_to_pa(*pte);
if (pa == 0) {
if (ARM_PTE_IS_COMPRESSED(*pte)) {
disp |= PMAP_QUERY_PAGE_COMPRESSED;
if (*pte & ARM_PTE_COMPRESSED_ALT) {
disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT;
}
}
} else {
disp |= PMAP_QUERY_PAGE_PRESENT;
pai = (int) pa_index(pa);
if (!pa_valid(pa)) {
goto done;
}
LOCK_PVH(pai);
pv_h = pai_to_pvh(pai);
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
while (pve_p != PV_ENTRY_NULL &&
pve_get_ptep(pve_p) != pte) {
pve_p = pvh_list(pv_h);
}
}
if (IS_ALTACCT_PAGE(pai, pve_p)) {
disp |= PMAP_QUERY_PAGE_ALTACCT;
} else if (IS_REUSABLE_PAGE(pai)) {
disp |= PMAP_QUERY_PAGE_REUSABLE;
} else if (IS_INTERNAL_PAGE(pai)) {
disp |= PMAP_QUERY_PAGE_INTERNAL;
}
UNLOCK_PVH(pai);
}
done:
PMAP_UNLOCK(pmap);
*disp_p = disp;
return KERN_SUCCESS;
}
kern_return_t
pmap_query_page_info(
pmap_t pmap,
vm_map_offset_t va,
int *disp_p)
{
return pmap_query_page_info_internal(pmap, va, disp_p);
}
kern_return_t
pmap_return_internal(__unused boolean_t do_panic, __unused boolean_t do_recurse)
{
return KERN_SUCCESS;
}
kern_return_t
pmap_return(boolean_t do_panic, boolean_t do_recurse)
{
return pmap_return_internal(do_panic, do_recurse);
}
static void
pmap_footprint_suspend_internal(
vm_map_t map,
boolean_t suspend)
{
#if DEVELOPMENT || DEBUG
if (suspend) {
map->pmap->footprint_suspended = TRUE;
map->pmap->footprint_was_suspended = TRUE;
} else {
map->pmap->footprint_suspended = FALSE;
}
#else
(void) map;
(void) suspend;
#endif
}
void
pmap_footprint_suspend(
vm_map_t map,
boolean_t suspend)
{
pmap_footprint_suspend_internal(map, suspend);
}