#include <string.h>
#include <mach_assert.h>
#include <mach_ldebug.h>
#include <mach/shared_region.h>
#include <mach/vm_param.h>
#include <mach/vm_prot.h>
#include <mach/vm_map.h>
#include <mach/machine/vm_param.h>
#include <mach/machine/vm_types.h>
#include <mach/boolean.h>
#include <kern/bits.h>
#include <kern/thread.h>
#include <kern/sched.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
#include <kern/ledger.h>
#include <kern/spl.h>
#include <kern/trustcache.h>
#include <os/overflow.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_protos.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/cpm.h>
#include <libkern/img4/interface.h>
#include <libkern/section_keywords.h>
#include <sys/errno.h>
#include <machine/atomic.h>
#include <machine/thread.h>
#include <machine/lowglobals.h>
#include <arm/caches_internal.h>
#include <arm/cpu_data.h>
#include <arm/cpu_data_internal.h>
#include <arm/cpu_capabilities.h>
#include <arm/cpu_number.h>
#include <arm/machine_cpu.h>
#include <arm/misc_protos.h>
#include <arm/trap.h>
#if (__ARM_VMSA__ > 7)
#include <arm64/proc_reg.h>
#include <pexpert/arm64/boot.h>
#if CONFIG_PGTRACE
#include <stdint.h>
#include <arm64/pgtrace.h>
#if CONFIG_PGTRACE_NONKEXT
#include <arm64/pgtrace_decoder.h>
#endif // CONFIG_PGTRACE_NONKEXT
#endif // CONFIG_PGTRACE
#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
#include <arm64/amcc_rorgn.h>
#endif // defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
#endif
#include <pexpert/device_tree.h>
#include <san/kasan.h>
#include <sys/cdefs.h>
#if defined(HAS_APPLE_PAC)
#include <ptrauth.h>
#endif
#ifdef CONFIG_XNUPOST
#include <tests/xnupost.h>
#endif
#if HIBERNATION
#include <IOKit/IOHibernatePrivate.h>
#endif
#define PMAP_TT_L0_LEVEL 0x0
#define PMAP_TT_L1_LEVEL 0x1
#define PMAP_TT_L2_LEVEL 0x2
#define PMAP_TT_L3_LEVEL 0x3
#ifdef __ARM64_PMAP_SUBPAGE_L1__
#if (__ARM_VMSA__ <= 7)
#error This is not supported for old-style page tables
#endif
#define PMAP_ROOT_ALLOC_SIZE (((ARM_TT_L1_INDEX_MASK >> ARM_TT_L1_SHIFT) + 1) * sizeof(tt_entry_t))
#else
#if (__ARM_VMSA__ <= 7)
#define PMAP_ROOT_ALLOC_SIZE (ARM_PGBYTES * 2)
#else
#define PMAP_ROOT_ALLOC_SIZE (ARM_PGBYTES)
#endif
#endif
extern u_int32_t random(void);
static bool alloc_asid(pmap_t pmap);
static void free_asid(pmap_t pmap);
static void flush_mmu_tlb_region_asid_async(vm_offset_t va, size_t length, pmap_t pmap);
static void flush_mmu_tlb_full_asid_async(pmap_t pmap);
static pt_entry_t wimg_to_pte(unsigned int wimg);
struct page_table_ops {
bool (*alloc_id)(pmap_t pmap);
void (*free_id)(pmap_t pmap);
void (*flush_tlb_region_async)(vm_offset_t va, size_t length, pmap_t pmap);
void (*flush_tlb_async)(pmap_t pmap);
pt_entry_t (*wimg_to_pte)(unsigned int wimg);
};
static const struct page_table_ops native_pt_ops =
{
.alloc_id = alloc_asid,
.free_id = free_asid,
.flush_tlb_region_async = flush_mmu_tlb_region_asid_async,
.flush_tlb_async = flush_mmu_tlb_full_asid_async,
.wimg_to_pte = wimg_to_pte,
};
#if (__ARM_VMSA__ > 7)
const struct page_table_level_info pmap_table_level_info_16k[] =
{
[0] = {
.size = ARM_16K_TT_L0_SIZE,
.offmask = ARM_16K_TT_L0_OFFMASK,
.shift = ARM_16K_TT_L0_SHIFT,
.index_mask = ARM_16K_TT_L0_INDEX_MASK,
.valid_mask = ARM_TTE_VALID,
.type_mask = ARM_TTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_BLOCK
},
[1] = {
.size = ARM_16K_TT_L1_SIZE,
.offmask = ARM_16K_TT_L1_OFFMASK,
.shift = ARM_16K_TT_L1_SHIFT,
.index_mask = ARM_16K_TT_L1_INDEX_MASK,
.valid_mask = ARM_TTE_VALID,
.type_mask = ARM_TTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_BLOCK
},
[2] = {
.size = ARM_16K_TT_L2_SIZE,
.offmask = ARM_16K_TT_L2_OFFMASK,
.shift = ARM_16K_TT_L2_SHIFT,
.index_mask = ARM_16K_TT_L2_INDEX_MASK,
.valid_mask = ARM_TTE_VALID,
.type_mask = ARM_TTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_BLOCK
},
[3] = {
.size = ARM_16K_TT_L3_SIZE,
.offmask = ARM_16K_TT_L3_OFFMASK,
.shift = ARM_16K_TT_L3_SHIFT,
.index_mask = ARM_16K_TT_L3_INDEX_MASK,
.valid_mask = ARM_PTE_TYPE_VALID,
.type_mask = ARM_PTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_L3BLOCK
}
};
const struct page_table_level_info pmap_table_level_info_4k[] =
{
[0] = {
.size = ARM_4K_TT_L0_SIZE,
.offmask = ARM_4K_TT_L0_OFFMASK,
.shift = ARM_4K_TT_L0_SHIFT,
.index_mask = ARM_4K_TT_L0_INDEX_MASK,
.valid_mask = ARM_TTE_VALID,
.type_mask = ARM_TTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_BLOCK
},
[1] = {
.size = ARM_4K_TT_L1_SIZE,
.offmask = ARM_4K_TT_L1_OFFMASK,
.shift = ARM_4K_TT_L1_SHIFT,
.index_mask = ARM_4K_TT_L1_INDEX_MASK,
.valid_mask = ARM_TTE_VALID,
.type_mask = ARM_TTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_BLOCK
},
[2] = {
.size = ARM_4K_TT_L2_SIZE,
.offmask = ARM_4K_TT_L2_OFFMASK,
.shift = ARM_4K_TT_L2_SHIFT,
.index_mask = ARM_4K_TT_L2_INDEX_MASK,
.valid_mask = ARM_TTE_VALID,
.type_mask = ARM_TTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_BLOCK
},
[3] = {
.size = ARM_4K_TT_L3_SIZE,
.offmask = ARM_4K_TT_L3_OFFMASK,
.shift = ARM_4K_TT_L3_SHIFT,
.index_mask = ARM_4K_TT_L3_INDEX_MASK,
.valid_mask = ARM_PTE_TYPE_VALID,
.type_mask = ARM_PTE_TYPE_MASK,
.type_block = ARM_TTE_TYPE_L3BLOCK
}
};
struct page_table_attr {
const struct page_table_level_info * const pta_level_info;
const struct page_table_ops * const pta_ops;
const uintptr_t ap_ro;
const uintptr_t ap_rw;
const uintptr_t ap_rona;
const uintptr_t ap_rwna;
const uintptr_t ap_xn;
const uintptr_t ap_x;
const unsigned int pta_root_level;
const unsigned int pta_sharedpage_level;
const unsigned int pta_max_level;
#if __ARM_MIXED_PAGE_SIZE__
const uint64_t pta_tcr_value;
#endif
const uint64_t pta_page_size;
const uint64_t pta_page_shift;
};
const struct page_table_attr pmap_pt_attr_4k = {
.pta_level_info = pmap_table_level_info_4k,
.pta_root_level = (T0SZ_BOOT - 16) / 9,
#if __ARM_MIXED_PAGE_SIZE__
.pta_sharedpage_level = PMAP_TT_L2_LEVEL,
#else
#if __ARM_16K_PG__
.pta_sharedpage_level = PMAP_TT_L2_LEVEL,
#else
.pta_sharedpage_level = PMAP_TT_L1_LEVEL,
#endif
#endif
.pta_max_level = PMAP_TT_L3_LEVEL,
.pta_ops = &native_pt_ops,
.ap_ro = ARM_PTE_AP(AP_RORO),
.ap_rw = ARM_PTE_AP(AP_RWRW),
.ap_rona = ARM_PTE_AP(AP_RONA),
.ap_rwna = ARM_PTE_AP(AP_RWNA),
.ap_xn = ARM_PTE_PNX | ARM_PTE_NX,
.ap_x = ARM_PTE_PNX,
#if __ARM_MIXED_PAGE_SIZE__
.pta_tcr_value = TCR_EL1_4KB,
#endif
.pta_page_size = 4096,
.pta_page_shift = 12,
};
const struct page_table_attr pmap_pt_attr_16k = {
.pta_level_info = pmap_table_level_info_16k,
.pta_root_level = PMAP_TT_L1_LEVEL,
.pta_sharedpage_level = PMAP_TT_L2_LEVEL,
.pta_max_level = PMAP_TT_L3_LEVEL,
.pta_ops = &native_pt_ops,
.ap_ro = ARM_PTE_AP(AP_RORO),
.ap_rw = ARM_PTE_AP(AP_RWRW),
.ap_rona = ARM_PTE_AP(AP_RONA),
.ap_rwna = ARM_PTE_AP(AP_RWNA),
.ap_xn = ARM_PTE_PNX | ARM_PTE_NX,
.ap_x = ARM_PTE_PNX,
#if __ARM_MIXED_PAGE_SIZE__
.pta_tcr_value = TCR_EL1_16KB,
#endif
.pta_page_size = 16384,
.pta_page_shift = 14,
};
#if __ARM_16K_PG__
const struct page_table_attr * const native_pt_attr = &pmap_pt_attr_16k;
#else
const struct page_table_attr * const native_pt_attr = &pmap_pt_attr_4k;
#endif
#else
const struct page_table_attr * const native_pt_attr = NULL;
#endif
typedef struct page_table_attr pt_attr_t;
#if ARM_PARAMETERIZED_PMAP
#define pmap_get_pt_attr(pmap) ((pmap)->pmap_pt_attr)
#define pmap_get_pt_ops(pmap) ((pmap)->pmap_pt_attr->pta_ops)
#else
#define pmap_get_pt_attr(pmap) (native_pt_attr)
#define pmap_get_pt_ops(pmap) (&native_pt_ops)
#endif
#if (__ARM_VMSA__ > 7)
static inline uint64_t
pt_attr_page_size(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_page_size;
}
__unused static inline uint64_t
pt_attr_ln_size(const pt_attr_t * const pt_attr, unsigned int level)
{
return pt_attr->pta_level_info[level].size;
}
__unused static inline uint64_t
pt_attr_ln_shift(const pt_attr_t * const pt_attr, unsigned int level)
{
return pt_attr->pta_level_info[level].shift;
}
static inline uint64_t
pt_attr_ln_offmask(const pt_attr_t * const pt_attr, unsigned int level)
{
return pt_attr->pta_level_info[level].offmask;
}
__unused static inline uint64_t
pt_attr_ln_pt_offmask(const pt_attr_t * const pt_attr, unsigned int level)
{
return pt_attr_ln_offmask(pt_attr, level);
}
__unused static inline uint64_t
pt_attr_ln_index_mask(const pt_attr_t * const pt_attr, unsigned int level)
{
return pt_attr->pta_level_info[level].index_mask;
}
static inline unsigned int
pt_attr_twig_level(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_max_level - 1;
}
static inline unsigned int
pt_attr_root_level(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_root_level;
}
static inline unsigned int
pt_attr_sharedpage_level(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_sharedpage_level;
}
static __unused inline uint64_t
pt_attr_leaf_size(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level].size;
}
static __unused inline uint64_t
pt_attr_leaf_offmask(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level].offmask;
}
static inline uint64_t
pt_attr_leaf_shift(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level].shift;
}
static __unused inline uint64_t
pt_attr_leaf_index_mask(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level].index_mask;
}
static inline uint64_t
pt_attr_twig_size(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].size;
}
static inline uint64_t
pt_attr_twig_offmask(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].offmask;
}
static inline uint64_t
pt_attr_twig_shift(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].shift;
}
static __unused inline uint64_t
pt_attr_twig_index_mask(const pt_attr_t * const pt_attr)
{
return pt_attr->pta_level_info[pt_attr->pta_max_level - 1].index_mask;
}
static inline uint64_t
pt_attr_leaf_table_size(const pt_attr_t * const pt_attr)
{
return pt_attr_twig_size(pt_attr);
}
static inline uint64_t
pt_attr_leaf_table_offmask(const pt_attr_t * const pt_attr)
{
return pt_attr_twig_offmask(pt_attr);
}
static inline uintptr_t
pt_attr_leaf_rw(const pt_attr_t * const pt_attr)
{
return pt_attr->ap_rw;
}
static inline uintptr_t
pt_attr_leaf_ro(const pt_attr_t * const pt_attr)
{
return pt_attr->ap_ro;
}
static inline uintptr_t
pt_attr_leaf_rona(const pt_attr_t * const pt_attr)
{
return pt_attr->ap_rona;
}
static inline uintptr_t
pt_attr_leaf_rwna(const pt_attr_t * const pt_attr)
{
return pt_attr->ap_rwna;
}
static inline uintptr_t
pt_attr_leaf_xn(const pt_attr_t * const pt_attr)
{
return pt_attr->ap_xn;
}
static inline uintptr_t
pt_attr_leaf_x(const pt_attr_t * const pt_attr)
{
return pt_attr->ap_x;
}
#else
static inline uint64_t
pt_attr_page_size(__unused const pt_attr_t * const pt_attr)
{
return PAGE_SIZE;
}
__unused static inline unsigned int
pt_attr_root_level(__unused const pt_attr_t * const pt_attr)
{
return PMAP_TT_L1_LEVEL;
}
__unused static inline unsigned int
pt_attr_sharedpage_level(__unused const pt_attr_t * const pt_attr)
{
return PMAP_TT_L1_LEVEL;
}
static inline unsigned int
pt_attr_twig_level(__unused const pt_attr_t * const pt_attr)
{
return PMAP_TT_L1_LEVEL;
}
static inline uint64_t
pt_attr_twig_size(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_TWIG_SIZE;
}
static inline uint64_t
pt_attr_twig_offmask(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_TWIG_OFFMASK;
}
static inline uint64_t
pt_attr_twig_shift(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_TWIG_SHIFT;
}
static __unused inline uint64_t
pt_attr_twig_index_mask(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_TWIG_INDEX_MASK;
}
__unused static inline uint64_t
pt_attr_leaf_size(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_LEAF_SIZE;
}
__unused static inline uint64_t
pt_attr_leaf_offmask(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_LEAF_OFFMASK;
}
static inline uint64_t
pt_attr_leaf_shift(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_LEAF_SHIFT;
}
static __unused inline uint64_t
pt_attr_leaf_index_mask(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_LEAF_INDEX_MASK;
}
static inline uint64_t
pt_attr_leaf_table_size(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_L1_PT_SIZE;
}
static inline uint64_t
pt_attr_leaf_table_offmask(__unused const pt_attr_t * const pt_attr)
{
return ARM_TT_L1_PT_OFFMASK;
}
static inline uintptr_t
pt_attr_leaf_rw(__unused const pt_attr_t * const pt_attr)
{
return ARM_PTE_AP(AP_RWRW);
}
static inline uintptr_t
pt_attr_leaf_ro(__unused const pt_attr_t * const pt_attr)
{
return ARM_PTE_AP(AP_RORO);
}
static inline uintptr_t
pt_attr_leaf_rona(__unused const pt_attr_t * const pt_attr)
{
return ARM_PTE_AP(AP_RONA);
}
static inline uintptr_t
pt_attr_leaf_rwna(__unused const pt_attr_t * const pt_attr)
{
return ARM_PTE_AP(AP_RWNA);
}
static inline uintptr_t
pt_attr_leaf_xn(__unused const pt_attr_t * const pt_attr)
{
return ARM_PTE_NX;
}
static inline uintptr_t
pt_attr_leaf_x(__unused const pt_attr_t * const pt_attr)
{
return ARM_PTE_PNX;
}
__unused static inline uintptr_t
pt_attr_ln_offmask(__unused const pt_attr_t * const pt_attr, unsigned int level)
{
if (level == PMAP_TT_L1_LEVEL) {
return ARM_TT_L1_OFFMASK;
} else if (level == PMAP_TT_L2_LEVEL) {
return ARM_TT_L2_OFFMASK;
}
return 0;
}
static inline uintptr_t
pt_attr_ln_pt_offmask(__unused const pt_attr_t * const pt_attr, unsigned int level)
{
if (level == PMAP_TT_L1_LEVEL) {
return ARM_TT_L1_PT_OFFMASK;
} else if (level == PMAP_TT_L2_LEVEL) {
return ARM_TT_L2_OFFMASK;
}
return 0;
}
#endif
static inline unsigned int
pt_attr_leaf_level(const pt_attr_t * const pt_attr)
{
return pt_attr_twig_level(pt_attr) + 1;
}
static inline void
pmap_sync_tlb(bool strong __unused)
{
sync_tlb_flush();
}
#if MACH_ASSERT
int vm_footprint_suspend_allowed = 1;
extern int pmap_ledgers_panic;
extern int pmap_ledgers_panic_leeway;
int pmap_stats_assert = 1;
#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...) \
MACRO_BEGIN \
if (pmap_stats_assert && (pmap)->pmap_stats_assert) \
assertf(cond, fmt, ##__VA_ARGS__); \
MACRO_END
#else
#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...)
#endif
#if DEVELOPMENT || DEBUG
#define PMAP_FOOTPRINT_SUSPENDED(pmap) \
(current_thread()->pmap_footprint_suspended)
#else
#define PMAP_FOOTPRINT_SUSPENDED(pmap) (FALSE)
#endif
#ifdef PLATFORM_BridgeOS
static struct pmap_legacy_trust_cache *pmap_legacy_trust_caches MARK_AS_PMAP_DATA = NULL;
#endif
static struct pmap_image4_trust_cache *pmap_image4_trust_caches MARK_AS_PMAP_DATA = NULL;
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pmap_loaded_trust_caches_lock, 0);
typedef struct pmap_tlb_flush_range {
pmap_t ptfr_pmap;
vm_map_address_t ptfr_start;
vm_map_address_t ptfr_end;
bool ptfr_flush_needed;
} pmap_tlb_flush_range_t;
#if XNU_MONITOR
extern vm_offset_t segPPLDATAB;
extern unsigned long segSizePPLDATA;
extern vm_offset_t segPPLTEXTB;
extern unsigned long segSizePPLTEXT;
extern vm_offset_t segPPLDATACONSTB;
extern unsigned long segSizePPLDATACONST;
#if (DEVELOPMENT || DEBUG) || CONFIG_CSR_FROM_DT
SECURITY_READ_ONLY_LATE(boolean_t) pmap_ppl_disable = FALSE;
#else
const boolean_t pmap_ppl_disable = FALSE;
#endif
boolean_t pmap_ppl_locked_down MARK_AS_PMAP_DATA = FALSE;
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pmap_ppl_free_page_lock, 0);
void ** pmap_ppl_free_page_list MARK_AS_PMAP_DATA = NULL;
uint64_t pmap_ppl_free_page_count MARK_AS_PMAP_DATA = 0;
uint64_t pmap_ppl_pages_returned_to_kernel_count_total = 0;
struct pmap_cpu_data_array_entry pmap_cpu_data_array[MAX_CPUS] MARK_AS_PMAP_DATA = {0};
extern void *pmap_stacks_start;
extern void *pmap_stacks_end;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) pmap_stacks_start_pa = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) pmap_stacks_end_pa = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) ppl_cpu_save_area_start = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) ppl_cpu_save_area_end = 0;
#if XNU_MONITOR
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pmap_free_list_lock, 0);
#endif
SECURITY_READ_ONLY_LATE(unsigned long) pmap_array_count = 0;
SECURITY_READ_ONLY_LATE(void *) pmap_array_begin = NULL;
SECURITY_READ_ONLY_LATE(void *) pmap_array_end = NULL;
SECURITY_READ_ONLY_LATE(pmap_t) pmap_array = NULL;
pmap_t pmap_free_list MARK_AS_PMAP_DATA = NULL;
#define PMAP_LEDGER_DATA_BYTES \
(((sizeof(task_ledgers) / sizeof(int)) * sizeof(struct ledger_entry)) + sizeof(struct ledger))
#define MAX_PMAP_LEDGERS (pmap_max_asids + 200)
#define PMAP_ARRAY_SIZE (pmap_max_asids)
typedef struct pmap_ledger_data {
char pld_data[PMAP_LEDGER_DATA_BYTES];
} pmap_ledger_data_t;
typedef struct pmap_ledger {
union {
struct pmap_ledger_data ple_data;
struct pmap_ledger * next;
};
struct pmap_ledger ** back_ptr;
} pmap_ledger_t;
SECURITY_READ_ONLY_LATE(bool) pmap_ledger_alloc_initialized = false;
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pmap_ledger_lock, 0);
SECURITY_READ_ONLY_LATE(void *) pmap_ledger_refcnt_begin = NULL;
SECURITY_READ_ONLY_LATE(void *) pmap_ledger_refcnt_end = NULL;
SECURITY_READ_ONLY_LATE(os_refcnt_t *) pmap_ledger_refcnt = NULL;
SECURITY_READ_ONLY_LATE(void *) pmap_ledger_ptr_array_begin = NULL;
SECURITY_READ_ONLY_LATE(void *) pmap_ledger_ptr_array_end = NULL;
SECURITY_READ_ONLY_LATE(pmap_ledger_t * *) pmap_ledger_ptr_array = NULL;
uint64_t pmap_ledger_ptr_array_free_index MARK_AS_PMAP_DATA = 0;
pmap_ledger_t * pmap_ledger_free_list MARK_AS_PMAP_DATA = NULL;
#define pmap_ledger_debit(p, e, a) ledger_debit_nocheck((p)->ledger, e, a)
#define pmap_ledger_credit(p, e, a) ledger_credit_nocheck((p)->ledger, e, a)
static inline void
pmap_check_ledger_fields(ledger_t ledger)
{
if (ledger == NULL) {
return;
}
thread_t cur_thread = current_thread();
ledger_check_new_balance(cur_thread, ledger, task_ledgers.alternate_accounting);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.alternate_accounting_compressed);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.internal);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.internal_compressed);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.page_table);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.phys_footprint);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.phys_mem);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.tkm_private);
ledger_check_new_balance(cur_thread, ledger, task_ledgers.wired_mem);
}
#define pmap_ledger_check_balance(p) pmap_check_ledger_fields((p)->ledger)
#else
#define pmap_ledger_debit(p, e, a) ledger_debit((p)->ledger, e, a)
#define pmap_ledger_credit(p, e, a) ledger_credit((p)->ledger, e, a)
#endif
#if (__ARM_VMSA__ == 7)
#define VREGION1_HIGH_WINDOW (0)
#else
#define VREGION1_HIGH_WINDOW (PE_EARLY_BOOT_VA)
#endif
#define VREGION1_START ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VREGION1_HIGH_WINDOW)
#define VREGION1_SIZE (trunc_page(VM_MAX_KERNEL_ADDRESS - (VREGION1_START)))
extern uint8_t bootstrap_pagetables[];
extern unsigned int not_in_kdp;
extern vm_offset_t first_avail;
extern pmap_paddr_t avail_start;
extern pmap_paddr_t avail_end;
extern vm_offset_t virtual_space_start;
extern vm_offset_t virtual_space_end;
extern vm_offset_t static_memory_end;
extern const vm_map_address_t physmap_base;
extern const vm_map_address_t physmap_end;
extern int maxproc, hard_maxproc;
vm_address_t MARK_AS_PMAP_DATA image4_slab = 0;
#if (__ARM_VMSA__ > 7)
#define PGTABLE_ADDR_BITS (64ULL - T0SZ_BOOT)
#define ARM64_TTBR0_MIN_ADDR (0ULL)
#define ARM64_TTBR0_MAX_ADDR (0ULL + (1ULL << PGTABLE_ADDR_BITS) - 1)
#define ARM64_TTBR1_MIN_ADDR (0ULL - (1ULL << PGTABLE_ADDR_BITS))
#define ARM64_TTBR1_MAX_ADDR (~0ULL)
const uint64_t arm64_root_pgtable_level = (3 - ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) / (ARM_PGSHIFT - TTE_SHIFT)));
const uint64_t arm64_root_pgtable_num_ttes = (2 << ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) % (ARM_PGSHIFT - TTE_SHIFT)));
#else
const uint64_t arm64_root_pgtable_level = 0;
const uint64_t arm64_root_pgtable_num_ttes = 0;
#endif
struct pmap kernel_pmap_store MARK_AS_PMAP_DATA;
SECURITY_READ_ONLY_LATE(pmap_t) kernel_pmap = &kernel_pmap_store;
struct vm_object pmap_object_store VM_PAGE_PACKED_ALIGNED;
SECURITY_READ_ONLY_LATE(vm_object_t) pmap_object = &pmap_object_store;
static SECURITY_READ_ONLY_LATE(zone_t) pmap_zone;
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pmaps_lock, 0);
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(tt1_lock, 0);
unsigned int pmap_stamp MARK_AS_PMAP_DATA;
queue_head_t map_pmap_list MARK_AS_PMAP_DATA;
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pt_pages_lock, 0);
queue_head_t pt_page_list MARK_AS_PMAP_DATA;
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pmap_pages_lock, 0);
typedef struct page_free_entry {
struct page_free_entry *next;
} page_free_entry_t;
#define PAGE_FREE_ENTRY_NULL ((page_free_entry_t *) 0)
page_free_entry_t *pmap_pages_reclaim_list MARK_AS_PMAP_DATA;
unsigned int pmap_pages_request_count MARK_AS_PMAP_DATA;
unsigned long long pmap_pages_request_acum MARK_AS_PMAP_DATA;
typedef struct tt_free_entry {
struct tt_free_entry *next;
} tt_free_entry_t;
#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0)
tt_free_entry_t *free_page_size_tt_list MARK_AS_PMAP_DATA;
unsigned int free_page_size_tt_count MARK_AS_PMAP_DATA;
unsigned int free_page_size_tt_max MARK_AS_PMAP_DATA;
#define FREE_PAGE_SIZE_TT_MAX 4
tt_free_entry_t *free_two_page_size_tt_list MARK_AS_PMAP_DATA;
unsigned int free_two_page_size_tt_count MARK_AS_PMAP_DATA;
unsigned int free_two_page_size_tt_max MARK_AS_PMAP_DATA;
#define FREE_TWO_PAGE_SIZE_TT_MAX 4
tt_free_entry_t *free_tt_list MARK_AS_PMAP_DATA;
unsigned int free_tt_count MARK_AS_PMAP_DATA;
unsigned int free_tt_max MARK_AS_PMAP_DATA;
#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0)
boolean_t pmap_gc_allowed MARK_AS_PMAP_DATA = TRUE;
boolean_t pmap_gc_forced MARK_AS_PMAP_DATA = FALSE;
boolean_t pmap_gc_allowed_by_time_throttle = TRUE;
unsigned int inuse_user_ttepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_user_ptepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_user_tteroot_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_kernel_ttepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_kernel_ptepages_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_kernel_tteroot_count MARK_AS_PMAP_DATA = 0;
unsigned int inuse_pmap_pages_count = 0;
SECURITY_READ_ONLY_LATE(tt_entry_t *) invalid_tte = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) invalid_ttep = 0;
SECURITY_READ_ONLY_LATE(tt_entry_t *) cpu_tte = 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) cpu_ttep = 0;
#if DEVELOPMENT || DEBUG
int nx_enabled = 1;
int allow_data_exec = 0;
int allow_stack_exec = 0;
unsigned long pmap_asid_flushes MARK_AS_PMAP_DATA = 0;
unsigned long pmap_asid_hits MARK_AS_PMAP_DATA = 0;
unsigned long pmap_asid_misses MARK_AS_PMAP_DATA = 0;
#else
const int nx_enabled = 1;
const int allow_data_exec = 0;
const int allow_stack_exec = 0;
#endif
bool hib_entry_pmap_lockdown MARK_AS_PMAP_DATA = false;
#if HIBERNATION
#define ASSERT_NOT_HIBERNATING() (assertf(!hib_entry_pmap_lockdown, \
"Attempted to modify PMAP data structures after hibernation image copying has begun."))
#else
#define ASSERT_NOT_HIBERNATING()
#endif
#define PV_ENTRY_NULL ((pv_entry_t *) 0)
#define PVE_NEXT_ALTACCT ((uintptr_t) 0x1)
#define PVE_NEXT_SET_ALTACCT(pve_next_p) \
*(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) | \
PVE_NEXT_ALTACCT)
#define PVE_NEXT_CLR_ALTACCT(pve_next_p) \
*(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) & \
~PVE_NEXT_ALTACCT)
#define PVE_NEXT_IS_ALTACCT(pve_next) \
((((uintptr_t) (pve_next)) & PVE_NEXT_ALTACCT) ? TRUE : FALSE)
#define PVE_NEXT_PTR(pve_next) \
((struct pv_entry *)(((uintptr_t) (pve_next)) & \
~PVE_NEXT_ALTACCT))
#if MACH_ASSERT
static void pmap_check_ledgers(pmap_t pmap);
#else
static inline void
pmap_check_ledgers(__unused pmap_t pmap)
{
}
#endif
SECURITY_READ_ONLY_LATE(pv_entry_t * *) pv_head_table;
pv_free_list_t pv_free MARK_AS_PMAP_DATA = {0};
pv_free_list_t pv_kern_free MARK_AS_PMAP_DATA = {0};
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pv_free_list_lock, 0);
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(pv_kern_free_list_lock, 0);
SIMPLE_LOCK_DECLARE(phys_backup_lock, 0);
#if (__ARM_VMSA__ == 7)
#define PT_INDEX_MAX 1
#else
#if __ARM_MIXED_PAGE_SIZE__
#define PT_INDEX_MAX (ARM_PGBYTES / 4096)
#elif (ARM_PGSHIFT == 14)
#define PT_INDEX_MAX 1
#elif (ARM_PGSHIFT == 12)
#define PT_INDEX_MAX 4
#else
#error Unsupported ARM_PGSHIFT
#endif
#endif
#define PT_DESC_REFCOUNT 0x4000U
#define PT_DESC_IOMMU_REFCOUNT 0x8000U
typedef struct {
unsigned short refcnt;
unsigned short wiredcnt;
vm_offset_t va;
} ptd_info_t;
typedef struct pt_desc {
queue_chain_t pt_page;
union {
struct pmap *pmap;
};
ptd_info_t ptd_info[PT_INDEX_MAX];
} pt_desc_t;
#define PTD_ENTRY_NULL ((pt_desc_t *) 0)
SECURITY_READ_ONLY_LATE(pt_desc_t *) ptd_root_table;
pt_desc_t *ptd_free_list MARK_AS_PMAP_DATA = PTD_ENTRY_NULL;
SECURITY_READ_ONLY_LATE(boolean_t) ptd_preboot = TRUE;
unsigned int ptd_free_count MARK_AS_PMAP_DATA = 0;
decl_simple_lock_data(, ptd_free_list_lock MARK_AS_PMAP_DATA);
typedef u_int16_t pp_attr_t;
#define PP_ATTR_WIMG_MASK 0x003F
#define PP_ATTR_WIMG(x) ((x) & PP_ATTR_WIMG_MASK)
#define PP_ATTR_REFERENCED 0x0040
#define PP_ATTR_MODIFIED 0x0080
#define PP_ATTR_INTERNAL 0x0100
#define PP_ATTR_REUSABLE 0x0200
#define PP_ATTR_ALTACCT 0x0400
#define PP_ATTR_NOENCRYPT 0x0800
#define PP_ATTR_REFFAULT 0x1000
#define PP_ATTR_MODFAULT 0x2000
#if XNU_MONITOR
#define PP_ATTR_MONITOR 0x4000
#define PP_ATTR_NO_MONITOR 0x8000
#define PP_ATTR_PPL_OWNED_BITS (PP_ATTR_MONITOR | PP_ATTR_NO_MONITOR)
#endif
SECURITY_READ_ONLY_LATE(volatile pp_attr_t*) pp_attr_table;
typedef struct pmap_io_range {
uint64_t addr;
uint64_t len;
#define PMAP_IO_RANGE_STRONG_SYNC (1UL << 31) #define PMAP_IO_RANGE_CARVEOUT (1UL << 30) #define PMAP_IO_RANGE_NEEDS_HIBERNATING (1UL << 29) uint32_t wimg; uint32_t signature; } __attribute__((packed)) pmap_io_range_t;
SECURITY_READ_ONLY_LATE(pmap_io_range_t*) io_attr_table = (pmap_io_range_t*)0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_first_phys = (pmap_paddr_t) 0;
SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_last_phys = (pmap_paddr_t) 0;
SECURITY_READ_ONLY_LATE(unsigned int) num_io_rgns = 0;
SECURITY_READ_ONLY_LATE(boolean_t) pmap_initialized = FALSE;
SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm_pmap_max_offset_default = 0x0;
#if defined(__arm64__)
# ifdef XNU_TARGET_OS_OSX
SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = MACH_VM_MAX_ADDRESS;
# else
SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0;
# endif
#endif
#if PMAP_PANIC_DEV_WIMG_ON_MANAGED && (DEVELOPMENT || DEBUG)
SECURITY_READ_ONLY_LATE(boolean_t) pmap_panic_dev_wimg_on_managed = TRUE;
#else
SECURITY_READ_ONLY_LATE(boolean_t) pmap_panic_dev_wimg_on_managed = FALSE;
#endif
MARK_AS_PMAP_DATA SIMPLE_LOCK_DECLARE(asid_lock, 0);
SECURITY_READ_ONLY_LATE(static uint32_t) pmap_max_asids = 0;
SECURITY_READ_ONLY_LATE(int) pmap_asid_plru = 1;
SECURITY_READ_ONLY_LATE(uint16_t) asid_chunk_size = 0;
SECURITY_READ_ONLY_LATE(static bitmap_t*) asid_bitmap;
static bitmap_t asid_plru_bitmap[BITMAP_LEN(MAX_HW_ASIDS)] MARK_AS_PMAP_DATA;
static uint64_t asid_plru_generation[BITMAP_LEN(MAX_HW_ASIDS)] MARK_AS_PMAP_DATA = {0};
static uint64_t asid_plru_gencount MARK_AS_PMAP_DATA = 0;
#if (__ARM_VMSA__ > 7)
#if __ARM_MIXED_PAGE_SIZE__
SECURITY_READ_ONLY_LATE(pmap_t) sharedpage_pmap_4k;
#endif
SECURITY_READ_ONLY_LATE(pmap_t) sharedpage_pmap_default;
#endif
#if XNU_MONITOR
#define PMAP_MIN_FREE_PPL_PAGES 8
#endif
#define pa_index(pa) \
(atop((pa) - vm_first_phys))
#define pai_to_pvh(pai) \
(&pv_head_table[pai])
#define pa_valid(x) \
((x) >= vm_first_phys && (x) < vm_last_phys)
#define pte_is_wired(pte) \
(((pte) & ARM_PTE_WIRED_MASK) == ARM_PTE_WIRED)
#define pte_was_writeable(pte) \
(((pte) & ARM_PTE_WRITEABLE) == ARM_PTE_WRITEABLE)
#define pte_set_was_writeable(pte, was_writeable) \
do { \
if ((was_writeable)) { \
(pte) |= ARM_PTE_WRITEABLE; \
} else { \
(pte) &= ~ARM_PTE_WRITEABLE; \
} \
} while(0)
#define pve_next(pve) \
((pve)->pve_next)
#define pve_link_field(pve) \
(&pve_next(pve))
#define pve_link(pp, e) \
((pve_next(e) = pve_next(pp)), (pve_next(pp) = (e)))
#define pve_unlink(pp, e) \
(pve_next(pp) = pve_next(e))
#define pve_get_ptep(pve) \
((pve)->pve_ptep)
#define pve_set_ptep(pve, ptep_new) \
do { \
(pve)->pve_ptep = (ptep_new); \
} while (0)
#define ARM_TT_PT_INDEX_MASK ARM_PGMASK
#if (__ARM_VMSA__ == 7)
#define ARM_TT_PT_ADDR_SHIFT (10U)
#define ptep_get_pmap(ptep) \
((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index(ml_static_vtop((((vm_offset_t)(ptep) & ~ARM_PGMASK))))))))->pmap))
#else
#if (ARM_PGSHIFT == 12)
#define ARM_TT_PT_ADDR_SHIFT (9ULL)
#else
#define ARM_TT_PT_ADDR_SHIFT (11ULL)
#endif
#define ptep_get_pmap(ptep) \
((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index(ml_static_vtop((((vm_offset_t)(ptep) & ~ARM_PGMASK))))))))->pmap))
#endif
#define ptep_get_ptd(ptep) \
((struct pt_desc *)(pvh_list(pai_to_pvh(pa_index(ml_static_vtop((vm_offset_t)(ptep)))))))
#define PVH_TYPE_NULL 0x0UL
#define PVH_TYPE_PVEP 0x1UL
#define PVH_TYPE_PTEP 0x2UL
#define PVH_TYPE_PTDP 0x3UL
#define PVH_TYPE_MASK (0x3UL)
#ifdef __arm64__
#define PVH_FLAG_IOMMU 0x4UL
#define PVH_FLAG_IOMMU_TABLE (1ULL << 63)
#define PVH_FLAG_CPU (1ULL << 62)
#define PVH_LOCK_BIT 61
#define PVH_FLAG_LOCK (1ULL << PVH_LOCK_BIT)
#define PVH_FLAG_EXEC (1ULL << 60)
#define PVH_FLAG_LOCKDOWN (1ULL << 59)
#define PVH_FLAG_HASHED (1ULL << 58)
#define PVH_HIGH_FLAGS (PVH_FLAG_CPU | PVH_FLAG_LOCK | PVH_FLAG_EXEC | PVH_FLAG_LOCKDOWN | PVH_FLAG_HASHED)
#else
#define PVH_LOCK_BIT 31
#define PVH_FLAG_LOCK (1UL << PVH_LOCK_BIT)
#define PVH_HIGH_FLAGS PVH_FLAG_LOCK
#endif
#define PVH_LIST_MASK (~PVH_TYPE_MASK)
#define pvh_test_type(h, b) \
((*(vm_offset_t *)(h) & (PVH_TYPE_MASK)) == (b))
#define pvh_ptep(h) \
((pt_entry_t *)((*(vm_offset_t *)(h) & PVH_LIST_MASK) | PVH_HIGH_FLAGS))
#define pvh_list(h) \
((pv_entry_t *)((*(vm_offset_t *)(h) & PVH_LIST_MASK) | PVH_HIGH_FLAGS))
#define pvh_get_flags(h) \
(*(vm_offset_t *)(h) & PVH_HIGH_FLAGS)
#define pvh_set_flags(h, f) \
do { \
os_atomic_store((vm_offset_t *)(h), (*(vm_offset_t *)(h) & ~PVH_HIGH_FLAGS) | (f), \
relaxed); \
} while (0)
#define pvh_update_head(h, e, t) \
do { \
assert(*(vm_offset_t *)(h) & PVH_FLAG_LOCK); \
os_atomic_store((vm_offset_t *)(h), (vm_offset_t)(e) | (t) | PVH_FLAG_LOCK, \
relaxed); \
} while (0)
#define pvh_update_head_unlocked(h, e, t) \
do { \
assert(!(*(vm_offset_t *)(h) & PVH_FLAG_LOCK)); \
*(vm_offset_t *)(h) = ((vm_offset_t)(e) | (t)) & ~PVH_FLAG_LOCK; \
} while (0)
#define pvh_add(h, e) \
do { \
assert(!pvh_test_type((h), PVH_TYPE_PTEP)); \
pve_next(e) = pvh_list(h); \
pvh_update_head((h), (e), PVH_TYPE_PVEP); \
} while (0)
#define pvh_remove(h, p, e) \
do { \
assert(!PVE_NEXT_IS_ALTACCT(pve_next((e)))); \
if ((p) == (h)) { \
if (PVE_NEXT_PTR(pve_next((e))) == PV_ENTRY_NULL) { \
pvh_update_head((h), PV_ENTRY_NULL, PVH_TYPE_NULL); \
} else { \
pvh_update_head((h), PVE_NEXT_PTR(pve_next((e))), PVH_TYPE_PVEP); \
} \
} else { \
\
boolean_t __is_altacct; \
__is_altacct = PVE_NEXT_IS_ALTACCT(*(p)); \
*(p) = PVE_NEXT_PTR(pve_next((e))); \
if (__is_altacct) { \
PVE_NEXT_SET_ALTACCT((p)); \
} else { \
PVE_NEXT_CLR_ALTACCT((p)); \
} \
} \
} while (0)
#define ppattr_set_bits(h, b) os_atomic_or((h), (pp_attr_t)(b), acq_rel)
#define ppattr_clear_bits(h, b) os_atomic_andnot((h), (pp_attr_t)(b), acq_rel)
#define ppattr_test_bits(h, b) \
((*(h) & (pp_attr_t)(b)) == (pp_attr_t)(b))
#define pa_set_bits(x, b) \
do { \
if (pa_valid(x)) \
ppattr_set_bits(&pp_attr_table[pa_index(x)], \
(b)); \
} while (0)
#define pa_test_bits(x, b) \
(pa_valid(x) ? ppattr_test_bits(&pp_attr_table[pa_index(x)],\
(b)) : FALSE)
#define pa_clear_bits(x, b) \
do { \
if (pa_valid(x)) \
ppattr_clear_bits(&pp_attr_table[pa_index(x)], \
(b)); \
} while (0)
#define pa_set_modify(x) \
pa_set_bits(x, PP_ATTR_MODIFIED)
#define pa_clear_modify(x) \
pa_clear_bits(x, PP_ATTR_MODIFIED)
#define pa_set_reference(x) \
pa_set_bits(x, PP_ATTR_REFERENCED)
#define pa_clear_reference(x) \
pa_clear_bits(x, PP_ATTR_REFERENCED)
#if XNU_MONITOR
#define pa_set_monitor(x) \
pa_set_bits((x), PP_ATTR_MONITOR)
#define pa_clear_monitor(x) \
pa_clear_bits((x), PP_ATTR_MONITOR)
#define pa_test_monitor(x) \
pa_test_bits((x), PP_ATTR_MONITOR)
#define pa_set_no_monitor(x) \
pa_set_bits((x), PP_ATTR_NO_MONITOR)
#define pa_clear_no_monitor(x) \
pa_clear_bits((x), PP_ATTR_NO_MONITOR)
#define pa_test_no_monitor(x) \
pa_test_bits((x), PP_ATTR_NO_MONITOR)
#endif
#define IS_INTERNAL_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
#define SET_INTERNAL_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
#define CLR_INTERNAL_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL)
#define IS_REUSABLE_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
#define SET_REUSABLE_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
#define CLR_REUSABLE_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE)
#define IS_ALTACCT_PAGE(pai, pve_p) \
(((pve_p) == NULL) \
? ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT) \
: PVE_NEXT_IS_ALTACCT(pve_next((pve_p))))
#define SET_ALTACCT_PAGE(pai, pve_p) \
if ((pve_p) == NULL) { \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT); \
} else { \
PVE_NEXT_SET_ALTACCT(&pve_next((pve_p))); \
}
#define CLR_ALTACCT_PAGE(pai, pve_p) \
if ((pve_p) == NULL) { \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT); \
} else { \
PVE_NEXT_CLR_ALTACCT(&pve_next((pve_p))); \
}
#define IS_REFFAULT_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
#define SET_REFFAULT_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
#define CLR_REFFAULT_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT)
#define IS_MODFAULT_PAGE(pai) \
ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
#define SET_MODFAULT_PAGE(pai) \
ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
#define CLR_MODFAULT_PAGE(pai) \
ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT)
#define tte_get_ptd(tte) \
((struct pt_desc *)(pvh_list(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK))))))
#if (__ARM_VMSA__ == 7)
#define tte_index(pmap, pt_attr, addr) \
ttenum((addr))
#define pte_index(pmap, pt_attr, addr) \
ptenum((addr))
#else
#define ttn_index(pmap, pt_attr, addr, pt_level) \
(((addr) & (pt_attr)->pta_level_info[(pt_level)].index_mask) >> (pt_attr)->pta_level_info[(pt_level)].shift)
#define tt0_index(pmap, pt_attr, addr) \
ttn_index((pmap), (pt_attr), (addr), PMAP_TT_L0_LEVEL)
#define tt1_index(pmap, pt_attr, addr) \
ttn_index((pmap), (pt_attr), (addr), PMAP_TT_L1_LEVEL)
#define tt2_index(pmap, pt_attr, addr) \
ttn_index((pmap), (pt_attr), (addr), PMAP_TT_L2_LEVEL)
#define tt3_index(pmap, pt_attr, addr) \
ttn_index((pmap), (pt_attr), (addr), PMAP_TT_L3_LEVEL)
#define tte_index(pmap, pt_attr, addr) \
tt2_index((pmap), (pt_attr), (addr))
#define pte_index(pmap, pt_attr, addr) \
tt3_index((pmap), (pt_attr), (addr))
#endif
static inline ptd_info_t *
ptd_get_info(pt_desc_t *ptd, const tt_entry_t *ttep)
{
assert(ptd->ptd_info[0].refcnt != PT_DESC_IOMMU_REFCOUNT);
#if PT_INDEX_MAX == 1
#pragma unused(ttep)
return &ptd->ptd_info[0];
#else
uint64_t pmap_page_shift = pt_attr_leaf_shift(pmap_get_pt_attr(ptd->pmap));
vm_offset_t ttep_page = (vm_offset_t)ttep >> pmap_page_shift;
unsigned int ttep_index = ttep_page & ((1U << (PAGE_SHIFT - pmap_page_shift)) - 1);
assert(ttep_index < PT_INDEX_MAX);
return &ptd->ptd_info[ttep_index];
#endif
}
static inline ptd_info_t *
ptep_get_info(const pt_entry_t *ptep)
{
return ptd_get_info(ptep_get_ptd(ptep), ptep);
}
static inline vm_map_address_t
ptep_get_va(const pt_entry_t *ptep)
{
pv_entry_t **pv_h;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(ptep_get_pmap(ptep));
pv_h = pai_to_pvh(pa_index(ml_static_vtop(((vm_offset_t)ptep))));;
assert(pvh_test_type(pv_h, PVH_TYPE_PTDP));
pt_desc_t *ptdp = (pt_desc_t *)(pvh_list(pv_h));
vm_map_address_t va = ptd_get_info(ptdp, ptep)->va;
vm_offset_t ptep_index = ((vm_offset_t)ptep & pt_attr_leaf_offmask(pt_attr)) / sizeof(*ptep);
va += (ptep_index << pt_attr_leaf_shift(pt_attr));
return va;
}
static inline void
pte_set_wired(pmap_t pmap, pt_entry_t *ptep, boolean_t wired)
{
if (wired) {
*ptep |= ARM_PTE_WIRED;
} else {
*ptep &= ~ARM_PTE_WIRED;
}
if (pmap == kernel_pmap) {
return;
}
unsigned short *ptd_wiredcnt_ptr;
ptd_wiredcnt_ptr = &(ptep_get_info(ptep)->wiredcnt);
if (wired) {
os_atomic_add(ptd_wiredcnt_ptr, (unsigned short)1, relaxed);
} else {
unsigned short prev_wired = os_atomic_sub_orig(ptd_wiredcnt_ptr, (unsigned short)1, relaxed);
if (__improbable(prev_wired == 0)) {
panic("pmap %p (pte %p): wired count underflow", pmap, ptep);
}
}
}
lck_grp_t pmap_lck_grp MARK_AS_PMAP_DATA;
static inline void
pmap_lock_init(pmap_t pmap)
{
lck_rw_init(&pmap->rwlock, &pmap_lck_grp, 0);
pmap->rwlock.lck_rw_can_sleep = FALSE;
}
static inline void
pmap_lock_destroy(pmap_t pmap)
{
lck_rw_destroy(&pmap->rwlock, &pmap_lck_grp);
}
static inline void
pmap_lock(pmap_t pmap)
{
#if !XNU_MONITOR
mp_disable_preemption();
#endif
lck_rw_lock_exclusive(&pmap->rwlock);
}
static inline void
pmap_lock_ro(pmap_t pmap)
{
#if !XNU_MONITOR
mp_disable_preemption();
#endif
lck_rw_lock_shared(&pmap->rwlock);
}
static inline void
pmap_unlock(pmap_t pmap)
{
lck_rw_unlock_exclusive(&pmap->rwlock);
#if !XNU_MONITOR
mp_enable_preemption();
#endif
}
static inline void
pmap_unlock_ro(pmap_t pmap)
{
lck_rw_unlock_shared(&pmap->rwlock);
#if !XNU_MONITOR
mp_enable_preemption();
#endif
}
static inline bool
pmap_try_lock(pmap_t pmap)
{
bool ret;
#if !XNU_MONITOR
mp_disable_preemption();
#endif
ret = lck_rw_try_lock_exclusive(&pmap->rwlock);
if (!ret) {
#if !XNU_MONITOR
mp_enable_preemption();
#endif
}
return ret;
}
__unused static inline void
pmap_assert_locked_r(__unused pmap_t pmap)
{
#if MACH_ASSERT
lck_rw_assert(&pmap->rwlock, LCK_RW_ASSERT_SHARED);
#else
(void)pmap;
#endif
}
__unused static inline void
pmap_assert_locked_w(__unused pmap_t pmap)
{
#if MACH_ASSERT
lck_rw_assert(&pmap->rwlock, LCK_RW_ASSERT_EXCLUSIVE);
#else
(void)pmap;
#endif
}
__unused static inline void
pmap_assert_locked_any(__unused pmap_t pmap)
{
#if MACH_ASSERT
lck_rw_assert(&pmap->rwlock, LCK_RW_ASSERT_HELD);
#endif
}
#if defined(__arm64__)
#define PVH_LOCK_WORD 1
#else
#define PVH_LOCK_WORD 0
#endif
#define ASSERT_PVH_LOCKED(index) \
do { \
assert((vm_offset_t)(pv_head_table[index]) & PVH_FLAG_LOCK); \
} while (0)
#define LOCK_PVH(index) \
do { \
pmap_lock_bit((uint32_t*)(&pv_head_table[index]) + PVH_LOCK_WORD, PVH_LOCK_BIT - (PVH_LOCK_WORD * 32)); \
} while (0)
#define UNLOCK_PVH(index) \
do { \
ASSERT_PVH_LOCKED(index); \
pmap_unlock_bit((uint32_t*)(&pv_head_table[index]) + PVH_LOCK_WORD, PVH_LOCK_BIT - (PVH_LOCK_WORD * 32)); \
} while (0)
#define PMAP_UPDATE_TLBS(pmap, s, e, strong) { \
pmap_get_pt_ops(pmap)->flush_tlb_region_async(s, (size_t)((e) - (s)), pmap); \
pmap_sync_tlb(strong); \
}
#define FLUSH_PTE_RANGE(spte, epte) \
__builtin_arm_dmb(DMB_ISH);
#define FLUSH_PTE(pte_p) \
__builtin_arm_dmb(DMB_ISH);
#define FLUSH_PTE_STRONG(pte_p) \
__builtin_arm_dsb(DSB_ISH);
#define FLUSH_PTE_RANGE_STRONG(spte, epte) \
__builtin_arm_dsb(DSB_ISH);
#define WRITE_PTE_FAST(pte_p, pte_entry) \
__unreachable_ok_push \
if (TEST_PAGE_RATIO_4) { \
if (((unsigned)(pte_p)) & 0x1f) { \
panic("%s: WRITE_PTE_FAST is unaligned, " \
"pte_p=%p, pte_entry=%p", \
__FUNCTION__, \
pte_p, (void*)pte_entry); \
} \
if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \
*(pte_p) = (pte_entry); \
*((pte_p)+1) = (pte_entry); \
*((pte_p)+2) = (pte_entry); \
*((pte_p)+3) = (pte_entry); \
} else { \
*(pte_p) = (pte_entry); \
*((pte_p)+1) = (pte_entry) | 0x1000; \
*((pte_p)+2) = (pte_entry) | 0x2000; \
*((pte_p)+3) = (pte_entry) | 0x3000; \
} \
} else { \
*(pte_p) = (pte_entry); \
} \
__unreachable_ok_pop
#define WRITE_PTE(pte_p, pte_entry) \
WRITE_PTE_FAST(pte_p, pte_entry); \
FLUSH_PTE(pte_p);
#define WRITE_PTE_STRONG(pte_p, pte_entry) \
WRITE_PTE_FAST(pte_p, pte_entry); \
FLUSH_PTE_STRONG(pte_p);
#define current_pmap() \
(vm_map_pmap(current_thread()->map))
#if XNU_MONITOR
#define ARRAY_ELEM_PTR_IS_VALID(_ptr_, _elem_size_, _array_begin_, _array_end_) \
(((_ptr_) >= (typeof(_ptr_))_array_begin_) && \
((_ptr_) < (typeof(_ptr_))_array_end_) && \
!((((void *)(_ptr_)) - ((void *)_array_begin_)) % (_elem_size_)))
#define PMAP_PTR_IS_VALID(x) ARRAY_ELEM_PTR_IS_VALID(x, sizeof(struct pmap), pmap_array_begin, pmap_array_end)
#define USER_PMAP_IS_VALID(x) (PMAP_PTR_IS_VALID(x) && (os_atomic_load(&(x)->ref_count, relaxed) > 0))
#define VALIDATE_PMAP(x) \
if (__improbable(((x) != kernel_pmap) && !USER_PMAP_IS_VALID(x))) \
panic("%s: invalid pmap %p", __func__, (x));
#define VALIDATE_LEDGER_PTR(x) \
if (__improbable(!ARRAY_ELEM_PTR_IS_VALID(x, sizeof(void *), pmap_ledger_ptr_array_begin, pmap_ledger_ptr_array_end))) \
panic("%s: invalid ledger ptr %p", __func__, (x));
#define ARRAY_ELEM_INDEX(x, _elem_size_, _array_begin_) ((uint64_t)((((void *)(x)) - (_array_begin_)) / (_elem_size_)))
static uint64_t
pmap_ledger_validate(void * ledger)
{
uint64_t array_index;
pmap_ledger_t ** ledger_ptr_array_ptr = ((pmap_ledger_t*)ledger)->back_ptr;
VALIDATE_LEDGER_PTR(ledger_ptr_array_ptr);
array_index = ARRAY_ELEM_INDEX(ledger_ptr_array_ptr, sizeof(pmap_ledger_t *), pmap_ledger_ptr_array_begin);
if (array_index >= MAX_PMAP_LEDGERS) {
panic("%s: ledger %p array index invalid, index was %#llx", __func__, ledger, array_index);
}
pmap_ledger_t *ledger_ptr = *ledger_ptr_array_ptr;
if (__improbable(ledger_ptr != ledger)) {
panic("%s: ledger pointer mismatch, %p != %p", __func__, ledger, ledger_ptr);
}
return array_index;
}
#else
#define VALIDATE_PMAP(x) assert((x) != NULL);
#endif
#if DEVELOPMENT || DEBUG
SECURITY_READ_ONLY_LATE(unsigned int) pmap_trace_mask = 0;
#define PMAP_TRACE(level, ...) \
if (__improbable((1 << (level)) & pmap_trace_mask)) { \
KDBG_RELEASE(__VA_ARGS__); \
}
#else
#define PMAP_TRACE(level, ...)
#endif
typedef enum {
PV_ALLOC_SUCCESS,
PV_ALLOC_RETRY,
PV_ALLOC_FAIL
} pv_alloc_return_t;
static pv_alloc_return_t pv_alloc(
pmap_t pmap,
unsigned int pai,
pv_entry_t **pvepp);
static void ptd_bootstrap(
pt_desc_t *ptdp, unsigned int ptd_cnt);
static inline pt_desc_t *ptd_alloc_unlinked(void);
static pt_desc_t *ptd_alloc(pmap_t pmap);
static void ptd_deallocate(pt_desc_t *ptdp);
static void ptd_init(
pt_desc_t *ptdp, pmap_t pmap, vm_map_address_t va, unsigned int ttlevel, pt_entry_t * pte_p);
static void pmap_set_reference(
ppnum_t pn);
pmap_paddr_t pmap_vtophys(
pmap_t pmap, addr64_t va);
void pmap_switch_user_ttb(
pmap_t pmap);
static kern_return_t pmap_expand(
pmap_t, vm_map_address_t, unsigned int options, unsigned int level);
static int pmap_remove_range(
pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *);
static int pmap_remove_range_options(
pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, vm_map_address_t *, bool *, int);
static tt_entry_t *pmap_tt1_allocate(
pmap_t, vm_size_t, unsigned int);
#define PMAP_TT_ALLOCATE_NOWAIT 0x1
static void pmap_tt1_deallocate(
pmap_t, tt_entry_t *, vm_size_t, unsigned int);
#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1
static kern_return_t pmap_tt_allocate(
pmap_t, tt_entry_t **, unsigned int, unsigned int);
#define PMAP_TT_ALLOCATE_NOWAIT 0x1
static void pmap_tte_deallocate(
pmap_t, vm_offset_t, vm_offset_t, bool, tt_entry_t *, unsigned int);
const unsigned int arm_hardware_page_size = ARM_PGBYTES;
const unsigned int arm_pt_desc_size = sizeof(pt_desc_t);
const unsigned int arm_pt_root_size = PMAP_ROOT_ALLOC_SIZE;
#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1
#if (__ARM_VMSA__ > 7)
static inline tt_entry_t *pmap_tt1e(
pmap_t, vm_map_address_t);
static inline tt_entry_t *pmap_tt2e(
pmap_t, vm_map_address_t);
static inline pt_entry_t *pmap_tt3e(
pmap_t, vm_map_address_t);
static inline pt_entry_t *pmap_ttne(
pmap_t, unsigned int, vm_map_address_t);
static void pmap_unmap_sharedpage(
pmap_t pmap);
static boolean_t
pmap_is_64bit(pmap_t);
#endif
static inline tt_entry_t *pmap_tte(
pmap_t, vm_map_address_t);
static inline pt_entry_t *pmap_pte(
pmap_t, vm_map_address_t);
static void pmap_update_cache_attributes_locked(
ppnum_t, unsigned);
static boolean_t arm_clear_fast_fault(
ppnum_t ppnum,
vm_prot_t fault_type);
static pmap_paddr_t pmap_pages_reclaim(
void);
static kern_return_t pmap_pages_alloc_zeroed(
pmap_paddr_t *pa,
unsigned size,
unsigned option);
#define PMAP_PAGES_ALLOCATE_NOWAIT 0x1
#define PMAP_PAGES_RECLAIM_NOWAIT 0x2
static void pmap_pages_free(
pmap_paddr_t pa,
unsigned size);
static void pmap_pin_kernel_pages(vm_offset_t kva, size_t nbytes);
static void pmap_unpin_kernel_pages(vm_offset_t kva, size_t nbytes);
static void pmap_trim_self(pmap_t pmap);
static void pmap_trim_subord(pmap_t subord);
static kern_return_t
pmap_enter_addr(
pmap_t pmap,
vm_map_address_t v,
pmap_paddr_t pa,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired);
kern_return_t
pmap_enter_options_addr(
pmap_t pmap,
vm_map_address_t v,
pmap_paddr_t pa,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options,
__unused void *arg);
#ifdef CONFIG_XNUPOST
kern_return_t pmap_test(void);
#endif
#if XNU_MONITOR
static pmap_paddr_t pmap_alloc_page_for_kern(unsigned int options);
static void pmap_alloc_page_for_ppl(unsigned int options);
#define GEN_ASM_NAME(__function_name) _##__function_name##_ppl
#define PMAP_SUPPORT_PROTOTYPES_WITH_ASM_INTERNAL(__return_type, __function_name, __function_args, __function_index, __assembly_function_name) \
static __return_type __function_name##_internal __function_args; \
extern __return_type __function_name##_ppl __function_args; \
__asm__ (".text \n" \
".align 2 \n" \
".globl " #__assembly_function_name "\n" \
#__assembly_function_name ":\n" \
"mov x15, " #__function_index "\n" \
"b _aprr_ppl_enter\n")
#define PMAP_SUPPORT_PROTOTYPES_WITH_ASM(__return_type, __function_name, __function_args, __function_index, __assembly_function_name) \
PMAP_SUPPORT_PROTOTYPES_WITH_ASM_INTERNAL(__return_type, __function_name, __function_args, __function_index, __assembly_function_name)
#define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \
PMAP_SUPPORT_PROTOTYPES_WITH_ASM(__return_type, __function_name, __function_args, __function_index, GEN_ASM_NAME(__function_name))
#else
#define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \
static __return_type __function_name##_internal __function_args
#endif
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
arm_fast_fault, (pmap_t pmap,
vm_map_address_t va,
vm_prot_t fault_type,
bool was_af_fault,
bool from_user), ARM_FAST_FAULT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
arm_force_fast_fault, (ppnum_t ppnum,
vm_prot_t allow_mode,
int options), ARM_FORCE_FAST_FAULT_INDEX);
MARK_AS_PMAP_TEXT static boolean_t
arm_force_fast_fault_with_flush_range(
ppnum_t ppnum,
vm_prot_t allow_mode,
int options,
pmap_tlb_flush_range_t *flush_range);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
mapping_free_prime, (void), MAPPING_FREE_PRIME_INDEX);
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
pmap_batch_set_cache_attributes, (ppnum_t pn,
unsigned int cacheattr,
unsigned int page_cnt,
unsigned int page_index,
boolean_t doit,
unsigned int *res), PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_change_wiring, (pmap_t pmap,
vm_map_address_t v,
boolean_t wired), PMAP_CHANGE_WIRING_INDEX);
PMAP_SUPPORT_PROTOTYPES(
pmap_t,
pmap_create_options, (ledger_t ledger,
vm_map_size_t size,
unsigned int flags,
kern_return_t * kr), PMAP_CREATE_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_destroy, (pmap_t pmap), PMAP_DESTROY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_enter_options, (pmap_t pmap,
vm_map_address_t v,
pmap_paddr_t pa,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options), PMAP_ENTER_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
pmap_paddr_t,
pmap_find_pa, (pmap_t pmap,
addr64_t va), PMAP_FIND_PA_INDEX);
#if (__ARM_VMSA__ > 7)
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_insert_sharedpage, (pmap_t pmap), PMAP_INSERT_SHAREDPAGE_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
boolean_t,
pmap_is_empty, (pmap_t pmap,
vm_map_offset_t va_start,
vm_map_offset_t va_end), PMAP_IS_EMPTY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
unsigned int,
pmap_map_cpu_windows_copy, (ppnum_t pn,
vm_prot_t prot,
unsigned int wimg_bits), PMAP_MAP_CPU_WINDOWS_COPY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_nest, (pmap_t grand,
pmap_t subord,
addr64_t vstart,
uint64_t size), PMAP_NEST_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_page_protect_options, (ppnum_t ppnum,
vm_prot_t prot,
unsigned int options,
void *arg), PMAP_PAGE_PROTECT_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
vm_map_address_t,
pmap_protect_options, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
vm_prot_t prot,
unsigned int options,
void *args), PMAP_PROTECT_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_query_page_info, (pmap_t pmap,
vm_map_offset_t va,
int *disp_p), PMAP_QUERY_PAGE_INFO_INDEX);
PMAP_SUPPORT_PROTOTYPES(
mach_vm_size_t,
pmap_query_resident, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
mach_vm_size_t * compressed_bytes_p), PMAP_QUERY_RESIDENT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_reference, (pmap_t pmap), PMAP_REFERENCE_INDEX);
PMAP_SUPPORT_PROTOTYPES(
vm_map_address_t,
pmap_remove_options, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
int options), PMAP_REMOVE_OPTIONS_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_return, (boolean_t do_panic,
boolean_t do_recurse), PMAP_RETURN_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_cache_attributes, (ppnum_t pn,
unsigned int cacheattr), PMAP_SET_CACHE_ATTRIBUTES_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_update_compressor_page, (ppnum_t pn,
unsigned int prev_cacheattr, unsigned int new_cacheattr), PMAP_UPDATE_COMPRESSOR_PAGE_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_nested, (pmap_t pmap), PMAP_SET_NESTED_INDEX);
#if MACH_ASSERT || XNU_MONITOR
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_process, (pmap_t pmap,
int pid,
char *procname), PMAP_SET_PROCESS_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_unmap_cpu_windows_copy, (unsigned int index), PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX);
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_unnest_options, (pmap_t grand,
addr64_t vaddr,
uint64_t size,
unsigned int option), PMAP_UNNEST_OPTIONS_INDEX);
#if XNU_MONITOR
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_cpu_data_init, (unsigned int cpu_number), PMAP_CPU_DATA_INIT_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
phys_attribute_set, (ppnum_t pn,
unsigned int bits), PHYS_ATTRIBUTE_SET_INDEX);
#if XNU_MONITOR
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_mark_page_as_ppl_page, (pmap_paddr_t pa, bool initially_free), PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
phys_attribute_clear, (ppnum_t pn,
unsigned int bits,
int options,
void *arg), PHYS_ATTRIBUTE_CLEAR_INDEX);
#if __ARM_RANGE_TLBI__
PMAP_SUPPORT_PROTOTYPES(
vm_map_address_t,
phys_attribute_clear_range, (pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
unsigned int bits,
unsigned int options), PHYS_ATTRIBUTE_CLEAR_RANGE_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_switch, (pmap_t pmap), PMAP_SWITCH_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_switch_user_ttb, (pmap_t pmap), PMAP_SWITCH_USER_TTB_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_clear_user_ttb, (void), PMAP_CLEAR_USER_TTB_INDEX);
#if XNU_MONITOR
PMAP_SUPPORT_PROTOTYPES(
uint64_t,
pmap_release_ppl_pages_to_kernel, (void), PMAP_RELEASE_PAGES_TO_KERNEL_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_vm_map_cs_enforced, (pmap_t pmap, bool new_value), PMAP_SET_VM_MAP_CS_ENFORCED_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_jit_entitled, (pmap_t pmap), PMAP_SET_JIT_ENTITLED_INDEX);
#if __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX)
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_disable_user_jop, (pmap_t pmap), PMAP_DISABLE_USER_JOP_INDEX);
#endif
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_trim, (pmap_t grand,
pmap_t subord,
addr64_t vstart,
uint64_t size), PMAP_TRIM_INDEX);
#if HAS_APPLE_PAC
PMAP_SUPPORT_PROTOTYPES(
void *,
pmap_sign_user_ptr, (void *value, ptrauth_key key, uint64_t discriminator, uint64_t jop_key), PMAP_SIGN_USER_PTR);
PMAP_SUPPORT_PROTOTYPES(
void *,
pmap_auth_user_ptr, (void *value, ptrauth_key key, uint64_t discriminator, uint64_t jop_key), PMAP_AUTH_USER_PTR);
#endif
PMAP_SUPPORT_PROTOTYPES(
bool,
pmap_is_trust_cache_loaded, (const uuid_t uuid), PMAP_IS_TRUST_CACHE_LOADED_INDEX);
PMAP_SUPPORT_PROTOTYPES(
uint32_t,
pmap_lookup_in_static_trust_cache, (const uint8_t cdhash[CS_CDHASH_LEN]), PMAP_LOOKUP_IN_STATIC_TRUST_CACHE_INDEX);
PMAP_SUPPORT_PROTOTYPES(
bool,
pmap_lookup_in_loaded_trust_caches, (const uint8_t cdhash[CS_CDHASH_LEN]), PMAP_LOOKUP_IN_LOADED_TRUST_CACHES_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_set_compilation_service_cdhash, (const uint8_t cdhash[CS_CDHASH_LEN]),
PMAP_SET_COMPILATION_SERVICE_CDHASH_INDEX);
PMAP_SUPPORT_PROTOTYPES(
bool,
pmap_match_compilation_service_cdhash, (const uint8_t cdhash[CS_CDHASH_LEN]),
PMAP_MATCH_COMPILATION_SERVICE_CDHASH_INDEX);
#if XNU_MONITOR
static void pmap_mark_page_as_ppl_page(pmap_paddr_t pa);
#endif
void pmap_footprint_suspend(vm_map_t map,
boolean_t suspend);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_footprint_suspend, (vm_map_t map,
boolean_t suspend),
PMAP_FOOTPRINT_SUSPEND_INDEX);
#if XNU_MONITOR
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_ledger_alloc_init, (size_t),
PMAP_LEDGER_ALLOC_INIT_INDEX);
PMAP_SUPPORT_PROTOTYPES(
ledger_t,
pmap_ledger_alloc, (void),
PMAP_LEDGER_ALLOC_INDEX);
PMAP_SUPPORT_PROTOTYPES(
void,
pmap_ledger_free, (ledger_t),
PMAP_LEDGER_FREE_INDEX);
#endif
#if CONFIG_PGTRACE
boolean_t pgtrace_enabled = 0;
typedef struct {
queue_chain_t chain;
pmap_t pmap;
vm_map_offset_t ova;
vm_map_offset_t cva[3];
pt_entry_t cva_spte[3];
struct {
pmap_paddr_t start;
pmap_paddr_t end;
} range;
bool cloned;
} pmap_pgtrace_map_t;
static void pmap_pgtrace_init(void);
static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end);
static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa_page, vm_map_offset_t va_page);
static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa);
#endif
#if DEVELOPMENT || DEBUG
PMAP_SUPPORT_PROTOTYPES(
kern_return_t,
pmap_test_text_corruption, (pmap_paddr_t),
PMAP_TEST_TEXT_CORRUPTION_INDEX);
#endif
#if (__ARM_VMSA__ > 7)
#if (ARM_PGSHIFT == 14)
#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x4000)
#else
#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x2000)
#endif
#else
#define LOWGLOBAL_ALIAS (0xFFFF1000)
#endif
long long alloc_tteroot_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
long long alloc_ttepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
long long alloc_ptepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL;
long long alloc_pmap_pages_count __attribute__((aligned(8))) = 0LL;
#if XNU_MONITOR
#if __has_feature(ptrauth_calls)
#define __ptrauth_ppl_handler __ptrauth(ptrauth_key_function_pointer, true, 0)
#else
#define __ptrauth_ppl_handler
#endif
const void * __ptrauth_ppl_handler const ppl_handler_table[PMAP_COUNT] = {
[ARM_FAST_FAULT_INDEX] = arm_fast_fault_internal,
[ARM_FORCE_FAST_FAULT_INDEX] = arm_force_fast_fault_internal,
[MAPPING_FREE_PRIME_INDEX] = mapping_free_prime_internal,
[PHYS_ATTRIBUTE_CLEAR_INDEX] = phys_attribute_clear_internal,
[PHYS_ATTRIBUTE_SET_INDEX] = phys_attribute_set_internal,
[PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX] = pmap_batch_set_cache_attributes_internal,
[PMAP_CHANGE_WIRING_INDEX] = pmap_change_wiring_internal,
[PMAP_CREATE_INDEX] = pmap_create_options_internal,
[PMAP_DESTROY_INDEX] = pmap_destroy_internal,
[PMAP_ENTER_OPTIONS_INDEX] = pmap_enter_options_internal,
[PMAP_FIND_PA_INDEX] = pmap_find_pa_internal,
[PMAP_INSERT_SHAREDPAGE_INDEX] = pmap_insert_sharedpage_internal,
[PMAP_IS_EMPTY_INDEX] = pmap_is_empty_internal,
[PMAP_MAP_CPU_WINDOWS_COPY_INDEX] = pmap_map_cpu_windows_copy_internal,
[PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX] = pmap_mark_page_as_ppl_page_internal,
[PMAP_NEST_INDEX] = pmap_nest_internal,
[PMAP_PAGE_PROTECT_OPTIONS_INDEX] = pmap_page_protect_options_internal,
[PMAP_PROTECT_OPTIONS_INDEX] = pmap_protect_options_internal,
[PMAP_QUERY_PAGE_INFO_INDEX] = pmap_query_page_info_internal,
[PMAP_QUERY_RESIDENT_INDEX] = pmap_query_resident_internal,
[PMAP_REFERENCE_INDEX] = pmap_reference_internal,
[PMAP_REMOVE_OPTIONS_INDEX] = pmap_remove_options_internal,
[PMAP_RETURN_INDEX] = pmap_return_internal,
[PMAP_SET_CACHE_ATTRIBUTES_INDEX] = pmap_set_cache_attributes_internal,
[PMAP_UPDATE_COMPRESSOR_PAGE_INDEX] = pmap_update_compressor_page_internal,
[PMAP_SET_NESTED_INDEX] = pmap_set_nested_internal,
[PMAP_SET_PROCESS_INDEX] = pmap_set_process_internal,
[PMAP_SWITCH_INDEX] = pmap_switch_internal,
[PMAP_SWITCH_USER_TTB_INDEX] = pmap_switch_user_ttb_internal,
[PMAP_CLEAR_USER_TTB_INDEX] = pmap_clear_user_ttb_internal,
[PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX] = pmap_unmap_cpu_windows_copy_internal,
[PMAP_UNNEST_OPTIONS_INDEX] = pmap_unnest_options_internal,
[PMAP_FOOTPRINT_SUSPEND_INDEX] = pmap_footprint_suspend_internal,
[PMAP_CPU_DATA_INIT_INDEX] = pmap_cpu_data_init_internal,
[PMAP_RELEASE_PAGES_TO_KERNEL_INDEX] = pmap_release_ppl_pages_to_kernel_internal,
[PMAP_SET_VM_MAP_CS_ENFORCED_INDEX] = pmap_set_vm_map_cs_enforced_internal,
[PMAP_SET_JIT_ENTITLED_INDEX] = pmap_set_jit_entitled_internal,
[PMAP_IS_TRUST_CACHE_LOADED_INDEX] = pmap_is_trust_cache_loaded_internal,
[PMAP_LOOKUP_IN_STATIC_TRUST_CACHE_INDEX] = pmap_lookup_in_static_trust_cache_internal,
[PMAP_LOOKUP_IN_LOADED_TRUST_CACHES_INDEX] = pmap_lookup_in_loaded_trust_caches_internal,
[PMAP_SET_COMPILATION_SERVICE_CDHASH_INDEX] = pmap_set_compilation_service_cdhash_internal,
[PMAP_MATCH_COMPILATION_SERVICE_CDHASH_INDEX] = pmap_match_compilation_service_cdhash_internal,
[PMAP_TRIM_INDEX] = pmap_trim_internal,
[PMAP_LEDGER_ALLOC_INIT_INDEX] = pmap_ledger_alloc_init_internal,
[PMAP_LEDGER_ALLOC_INDEX] = pmap_ledger_alloc_internal,
[PMAP_LEDGER_FREE_INDEX] = pmap_ledger_free_internal,
#if HAS_APPLE_PAC
[PMAP_SIGN_USER_PTR] = pmap_sign_user_ptr_internal,
[PMAP_AUTH_USER_PTR] = pmap_auth_user_ptr_internal,
#endif
#if __ARM_RANGE_TLBI__
[PHYS_ATTRIBUTE_CLEAR_RANGE_INDEX] = phys_attribute_clear_range_internal,
#endif
#if __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX)
[PMAP_DISABLE_USER_JOP_INDEX] = pmap_disable_user_jop_internal,
#endif
#if DEVELOPMENT || DEBUG
[PMAP_TEST_TEXT_CORRUPTION_INDEX] = pmap_test_text_corruption_internal,
#endif
};
#endif
MARK_AS_PMAP_TEXT static void
pmap_cpu_data_init_internal(unsigned int cpu_number)
{
pmap_cpu_data_t * pmap_cpu_data = pmap_get_cpu_data();
#if XNU_MONITOR
assert(((vm_offset_t)pmap_cpu_data & ((1 << MAX_L2_CLINE) - 1)) == 0);
if (pmap_cpu_data->cpu_number != PMAP_INVALID_CPU_NUM) {
panic("%s: pmap_cpu_data->cpu_number=%u, "
"cpu_number=%u",
__FUNCTION__, pmap_cpu_data->cpu_number,
cpu_number);
}
#endif
pmap_cpu_data->cpu_number = cpu_number;
}
void
pmap_cpu_data_init(void)
{
#if XNU_MONITOR
pmap_cpu_data_init_ppl(cpu_number());
#else
pmap_cpu_data_init_internal(cpu_number());
#endif
}
static void
pmap_cpu_data_array_init(void)
{
#if XNU_MONITOR
unsigned int i = 0;
pmap_paddr_t ppl_cpu_save_area_cur = 0;
pt_entry_t template, *pte_p;
vm_offset_t stack_va = (vm_offset_t)pmap_stacks_start + ARM_PGBYTES;
assert((pmap_stacks_start != NULL) && (pmap_stacks_end != NULL));
pmap_stacks_start_pa = avail_start;
for (i = 0; i < MAX_CPUS; i++) {
for (vm_offset_t cur_va = stack_va; cur_va < (stack_va + PPL_STACK_SIZE); cur_va += ARM_PGBYTES) {
assert(cur_va < (vm_offset_t)pmap_stacks_end);
pte_p = pmap_pte(kernel_pmap, cur_va);
assert(*pte_p == ARM_PTE_EMPTY);
template = pa_to_pte(avail_start) | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_TYPE |
ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) | xprr_perm_to_pte(XPRR_PPL_RW_PERM);
#if __ARM_KERNEL_PROTECT__
template |= ARM_PTE_NG;
#endif
WRITE_PTE(pte_p, template);
__builtin_arm_isb(ISB_SY);
avail_start += ARM_PGBYTES;
}
#if KASAN
kasan_map_shadow(stack_va, PPL_STACK_SIZE, false);
#endif
pmap_cpu_data_array[i].cpu_data.cpu_number = PMAP_INVALID_CPU_NUM;
pmap_cpu_data_array[i].cpu_data.ppl_state = PPL_STATE_KERNEL;
pmap_cpu_data_array[i].cpu_data.ppl_stack = (void*)(stack_va + PPL_STACK_SIZE);
stack_va += (PPL_STACK_SIZE + ARM_PGBYTES);
}
sync_tlb_flush();
pmap_stacks_end_pa = avail_start;
ppl_cpu_save_area_start = avail_start;
ppl_cpu_save_area_end = ppl_cpu_save_area_start;
ppl_cpu_save_area_cur = ppl_cpu_save_area_start;
for (i = 0; i < MAX_CPUS; i++) {
while ((ppl_cpu_save_area_end - ppl_cpu_save_area_cur) < sizeof(arm_context_t)) {
avail_start += PAGE_SIZE;
ppl_cpu_save_area_end = avail_start;
}
pmap_cpu_data_array[i].cpu_data.save_area = (arm_context_t *)phystokv(ppl_cpu_save_area_cur);
ppl_cpu_save_area_cur += sizeof(arm_context_t);
}
#endif
pmap_cpu_data_init();
}
pmap_cpu_data_t *
pmap_get_cpu_data(void)
{
pmap_cpu_data_t * pmap_cpu_data = NULL;
#if XNU_MONITOR
extern pmap_cpu_data_t* ml_get_ppl_cpu_data(void);
pmap_cpu_data = ml_get_ppl_cpu_data();
#else
pmap_cpu_data = &getCpuDatap()->cpu_pmap_cpu_data;
#endif
return pmap_cpu_data;
}
#if __arm64__
static uint64_t __attribute__((warn_unused_result)) __used
pmap_interrupts_disable(void)
{
uint64_t state = __builtin_arm_rsr64("DAIF");
if ((state & DAIF_STANDARD_DISABLE) != DAIF_STANDARD_DISABLE) {
__builtin_arm_wsr64("DAIFSet", DAIFSC_STANDARD_DISABLE);
}
return state;
}
static void __used
pmap_interrupts_restore(uint64_t state)
{
assert((state & ~DAIF_ALL) == 0);
if (state != DAIF_STANDARD_DISABLE) {
__builtin_arm_wsr64("DAIF", state);
}
}
static bool __attribute__((warn_unused_result)) __used
pmap_interrupts_enabled(void)
{
return (__builtin_arm_rsr64("DAIF") & DAIF_STANDARD_DISABLE) != DAIF_STANDARD_DISABLE;
}
#endif
#if XNU_MONITOR
static void
pmap_set_range_xprr_perm(vm_address_t start,
vm_address_t end,
unsigned int expected_perm,
unsigned int new_perm)
{
#if (__ARM_VMSA__ == 7)
#error This function is not supported on older ARM hardware
#else
pmap_t pmap = NULL;
vm_address_t va = 0;
vm_address_t tte_start = 0;
vm_address_t tte_end = 0;
tt_entry_t *tte_p = NULL;
pt_entry_t *pte_p = NULL;
pt_entry_t *cpte_p = NULL;
pt_entry_t *bpte_p = NULL;
pt_entry_t *epte_p = NULL;
tt_entry_t tte = 0;
pt_entry_t cpte = 0;
pt_entry_t template = 0;
pmap = kernel_pmap;
va = start;
if ((start | end) % ARM_PGBYTES) {
panic("%s: start or end not page aligned, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)start, (void *)end, new_perm, expected_perm);
}
if (start > end) {
panic("%s: start > end, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)start, (void *)end, new_perm, expected_perm);
}
bool in_physmap = (start >= physmap_base) && (end < physmap_end);
bool in_static = (start >= gVirtBase) && (end < static_memory_end);
if (!(in_physmap || in_static)) {
panic("%s: address not in static region or physical aperture, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)start, (void *)end, new_perm, expected_perm);
}
if ((new_perm > XPRR_MAX_PERM) || (expected_perm > XPRR_MAX_PERM)) {
panic("%s: invalid XPRR index, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)start, (void *)end, new_perm, expected_perm);
}
while (va < end) {
tte_start = va;
tte_end = ((va + pt_attr_twig_size(native_pt_attr)) & ~pt_attr_twig_offmask(native_pt_attr));
if (tte_end > end) {
tte_end = end;
}
tte_p = pmap_tte(pmap, va);
if (tte_p == NULL) {
panic("%s: physical aperture tte is NULL, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)start, (void *)end, new_perm, expected_perm);
}
tte = *tte_p;
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *)ttetokv(tte);
bpte_p = &pte_p[pte_index(pmap, native_pt_attr, va)];
epte_p = bpte_p + ((tte_end - va) >> pt_attr_leaf_shift(native_pt_attr));
for (cpte_p = bpte_p; cpte_p < epte_p;
cpte_p += PAGE_SIZE / ARM_PGBYTES, va += PAGE_SIZE) {
int pai = (int)pa_index(pte_to_pa(*cpte_p));
LOCK_PVH(pai);
cpte = *cpte_p;
if ((cpte & ARM_PTE_TYPE_MASK) ==
ARM_PTE_TYPE_FAULT) {
panic("%s: physical aperture PTE is invalid, va=%p, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)va,
(void *)start, (void *)end, new_perm, expected_perm);
UNLOCK_PVH(pai);
continue;
}
if (cpte & ARM_PTE_HINT_MASK) {
panic("%s: physical aperture PTE has hint bit set, va=%p, cpte=0x%llx, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
(void *)va, cpte,
(void *)start, (void *)end, new_perm, expected_perm);
}
if (pte_to_xprr_perm(cpte) != expected_perm) {
panic("%s: perm=%llu does not match expected_perm, cpte=0x%llx, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
pte_to_xprr_perm(cpte), cpte,
(void *)start, (void *)end, new_perm, expected_perm);
}
template = cpte;
template &= ~ARM_PTE_XPRR_MASK;
template |= xprr_perm_to_pte(new_perm);
WRITE_PTE_STRONG(cpte_p, template);
UNLOCK_PVH(pai);
}
} else {
panic("%s: tte=0x%llx is not a table type entry, "
"start=%p, end=%p, new_perm=%u, expected_perm=%u",
__FUNCTION__,
tte,
(void *)start, (void *)end, new_perm, expected_perm);
}
va = tte_end;
}
PMAP_UPDATE_TLBS(pmap, start, end, false);
#endif
}
static inline void
pmap_set_xprr_perm(vm_address_t page_kva,
unsigned int expected_perm,
unsigned int new_perm)
{
pmap_set_range_xprr_perm(page_kva, page_kva + PAGE_SIZE, expected_perm, new_perm);
}
#endif
pmap_paddr_t
pmap_pages_reclaim(
void)
{
boolean_t found_page;
unsigned i;
pt_desc_t *ptdp;
pmap_simple_lock(&pmap_pages_lock);
pmap_pages_request_count++;
pmap_pages_request_acum++;
while (1) {
if (pmap_pages_reclaim_list != (page_free_entry_t *)NULL) {
page_free_entry_t *page_entry;
page_entry = pmap_pages_reclaim_list;
pmap_pages_reclaim_list = pmap_pages_reclaim_list->next;
pmap_simple_unlock(&pmap_pages_lock);
return (pmap_paddr_t)ml_static_vtop((vm_offset_t)page_entry);
}
pmap_simple_unlock(&pmap_pages_lock);
pmap_simple_lock(&pt_pages_lock);
ptdp = (pt_desc_t *)queue_first(&pt_page_list);
found_page = FALSE;
while (!queue_end(&pt_page_list, (queue_entry_t)ptdp)) {
if ((ptdp->pmap->nested == FALSE)
&& (pmap_try_lock(ptdp->pmap))) {
assert(ptdp->pmap != kernel_pmap);
unsigned refcnt_acc = 0;
unsigned wiredcnt_acc = 0;
for (i = 0; i < PT_INDEX_MAX; i++) {
if (ptdp->ptd_info[i].refcnt == PT_DESC_REFCOUNT) {
refcnt_acc = 0;
break;
}
refcnt_acc += ptdp->ptd_info[i].refcnt;
wiredcnt_acc += ptdp->ptd_info[i].wiredcnt;
}
if ((wiredcnt_acc == 0) && (refcnt_acc != 0)) {
found_page = TRUE;
break;
}
pmap_unlock(ptdp->pmap);
}
ptdp = (pt_desc_t *)queue_next((queue_t)ptdp);
}
if (!found_page) {
pmap_simple_unlock(&pt_pages_lock);
return (pmap_paddr_t)0;
} else {
bool need_strong_sync = false;
vm_map_address_t va;
pmap_t pmap;
pt_entry_t *bpte, *epte;
pt_entry_t *pte_p;
tt_entry_t *tte_p;
pmap_simple_unlock(&pt_pages_lock);
pmap = ptdp->pmap;
pmap_assert_locked_w(pmap);
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
for (i = 0; i < (PAGE_SIZE / pt_attr_page_size(pt_attr)); i++) {
va = ptdp->ptd_info[i].va;
if (va == (vm_offset_t)-1) {
continue;
}
tte_p = pmap_tte(pmap, va);
if ((tte_p != (tt_entry_t *) NULL)
&& ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[pte_index(pmap, pt_attr, va)];
epte = bpte + pt_attr_page_size(pt_attr) / sizeof(pt_entry_t);
pmap_remove_range_options(
pmap, va, bpte, epte, NULL,
&need_strong_sync, PMAP_OPTIONS_REMOVE);
if (ptd_get_info(ptdp, pte_p)->refcnt != 0) {
panic("%s: ptdp %p, count %d", __FUNCTION__, ptdp, ptd_get_info(ptdp, pte_p)->refcnt);
}
pmap_tte_deallocate(pmap, va, va + (size_t)pt_attr_leaf_table_size(pt_attr), need_strong_sync,
tte_p, pt_attr_twig_level(pt_attr));
}
}
pmap_unlock(pmap);
}
pmap_simple_lock(&pmap_pages_lock);
}
}
#if XNU_MONITOR
MARK_AS_PMAP_TEXT static void
pmap_give_free_ppl_page(pmap_paddr_t paddr)
{
assert((paddr & ARM_PGMASK) == 0);
void ** new_head = (void **)phystokv(paddr);
pmap_simple_lock(&pmap_ppl_free_page_lock);
void * cur_head = pmap_ppl_free_page_list;
*new_head = cur_head;
pmap_ppl_free_page_list = new_head;
pmap_ppl_free_page_count++;
pmap_simple_unlock(&pmap_ppl_free_page_lock);
}
MARK_AS_PMAP_TEXT static pmap_paddr_t
pmap_get_free_ppl_page(void)
{
pmap_paddr_t result = 0;
pmap_simple_lock(&pmap_ppl_free_page_lock);
if (pmap_ppl_free_page_list != NULL) {
void ** new_head = NULL;
new_head = *((void**)pmap_ppl_free_page_list);
result = kvtophys((vm_offset_t)pmap_ppl_free_page_list);
pmap_ppl_free_page_list = new_head;
pmap_ppl_free_page_count--;
} else {
result = 0L;
}
pmap_simple_unlock(&pmap_ppl_free_page_lock);
assert((result & ARM_PGMASK) == 0);
return result;
}
MARK_AS_PMAP_TEXT static void
pmap_mark_page_as_ppl_page_internal(pmap_paddr_t pa, bool initially_free)
{
vm_offset_t kva = 0;
unsigned int pai = 0;
pp_attr_t attr;
if (!pa_valid(pa)) {
panic("%s: bad address, "
"pa=%p",
__func__,
(void *)pa);
}
pai = (unsigned int)pa_index(pa);
LOCK_PVH(pai);
if (pa_test_monitor(pa)) {
panic("%s: page already belongs to PPL, "
"pa=0x%llx",
__FUNCTION__,
pa);
}
if (!pmap_verify_free((ppnum_t)atop(pa))) {
panic("%s: page is not free, "
"pa=0x%llx",
__FUNCTION__,
pa);
}
do {
attr = pp_attr_table[pai];
if (attr & PP_ATTR_NO_MONITOR) {
panic("%s: page excluded from PPL, "
"pa=0x%llx",
__FUNCTION__,
pa);
}
} while (!OSCompareAndSwap16(attr, attr | PP_ATTR_MONITOR, &pp_attr_table[pai]));
UNLOCK_PVH(pai);
kva = phystokv(pa);
pmap_set_xprr_perm(kva, XPRR_KERN_RW_PERM, XPRR_PPL_RW_PERM);
if (initially_free) {
pmap_give_free_ppl_page(pa);
}
}
static void
pmap_mark_page_as_ppl_page(pmap_paddr_t pa)
{
pmap_mark_page_as_ppl_page_ppl(pa, true);
}
MARK_AS_PMAP_TEXT static void
pmap_mark_page_as_kernel_page(pmap_paddr_t pa)
{
vm_offset_t kva = 0;
unsigned int pai = 0;
pai = (unsigned int)pa_index(pa);
LOCK_PVH(pai);
if (!pa_test_monitor(pa)) {
panic("%s: page is not a PPL page, "
"pa=%p",
__FUNCTION__,
(void *)pa);
}
pa_clear_monitor(pa);
UNLOCK_PVH(pai);
kva = phystokv(pa);
pmap_set_xprr_perm(kva, XPRR_PPL_RW_PERM, XPRR_KERN_RW_PERM);
}
MARK_AS_PMAP_TEXT static pmap_paddr_t
pmap_release_ppl_pages_to_kernel_internal(void)
{
pmap_paddr_t pa = 0;
if (pmap_ppl_free_page_count <= PMAP_MIN_FREE_PPL_PAGES) {
goto done;
}
pa = pmap_get_free_ppl_page();
if (!pa) {
goto done;
}
pmap_mark_page_as_kernel_page(pa);
done:
return pa;
}
static uint64_t
pmap_release_ppl_pages_to_kernel(void)
{
pmap_paddr_t pa = 0;
vm_page_t m = VM_PAGE_NULL;
vm_page_t local_freeq = VM_PAGE_NULL;
uint64_t pmap_ppl_pages_returned_to_kernel_count = 0;
while (pmap_ppl_free_page_count > PMAP_MIN_FREE_PPL_PAGES) {
pa = pmap_release_ppl_pages_to_kernel_ppl();
if (!pa) {
break;
}
vm_object_lock(pmap_object);
m = vm_page_lookup(pmap_object, (pa - gPhysBase));
assert(m != VM_PAGE_NULL);
assert(VM_PAGE_WIRED(m));
m->vmp_busy = TRUE;
m->vmp_snext = local_freeq;
local_freeq = m;
pmap_ppl_pages_returned_to_kernel_count++;
pmap_ppl_pages_returned_to_kernel_count_total++;
vm_object_unlock(pmap_object);
}
if (local_freeq) {
vm_object_lock(pmap_object);
vm_page_free_list(local_freeq, TRUE);
vm_object_unlock(pmap_object);
}
return pmap_ppl_pages_returned_to_kernel_count;
}
#endif
static inline void
pmap_enqueue_pages(vm_page_t m)
{
vm_page_t m_prev;
vm_object_lock(pmap_object);
while (m != VM_PAGE_NULL) {
vm_page_insert_wired(m, pmap_object, (vm_object_offset_t) ((ptoa(VM_PAGE_GET_PHYS_PAGE(m))) - gPhysBase), VM_KERN_MEMORY_PTE);
m_prev = m;
m = NEXT_PAGE(m_prev);
*(NEXT_PAGE_PTR(m_prev)) = VM_PAGE_NULL;
}
vm_object_unlock(pmap_object);
}
static kern_return_t
pmap_pages_alloc_zeroed(
pmap_paddr_t *pa,
unsigned size,
unsigned option)
{
#if XNU_MONITOR
ASSERT_NOT_HIBERNATING();
if (size != PAGE_SIZE) {
panic("%s: size != PAGE_SIZE, "
"pa=%p, size=%u, option=%u",
__FUNCTION__,
pa, size, option);
}
assert(option & PMAP_PAGES_ALLOCATE_NOWAIT);
*pa = pmap_get_free_ppl_page();
if ((*pa == 0) && (option & PMAP_PAGES_RECLAIM_NOWAIT)) {
*pa = pmap_pages_reclaim();
}
if (*pa == 0) {
return KERN_RESOURCE_SHORTAGE;
} else {
bzero((void*)phystokv(*pa), size);
return KERN_SUCCESS;
}
#else
vm_page_t m = VM_PAGE_NULL;
thread_t self = current_thread();
uint16_t thread_options = self->options;
self->options |= TH_OPT_VMPRIV;
if (__probable(size == PAGE_SIZE)) {
while ((m = vm_page_grab()) == VM_PAGE_NULL) {
if (option & PMAP_PAGES_ALLOCATE_NOWAIT) {
break;
}
VM_PAGE_WAIT();
}
if (m != VM_PAGE_NULL) {
vm_page_lock_queues();
vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
vm_page_unlock_queues();
}
} else if (size == 2 * PAGE_SIZE) {
while (cpm_allocate(size, &m, 0, 1, TRUE, 0) != KERN_SUCCESS) {
if (option & PMAP_PAGES_ALLOCATE_NOWAIT) {
break;
}
VM_PAGE_WAIT();
}
} else {
panic("%s: invalid size %u", __func__, size);
}
self->options = thread_options;
if ((m == VM_PAGE_NULL) && (option & PMAP_PAGES_RECLAIM_NOWAIT)) {
assert(size == PAGE_SIZE);
*pa = pmap_pages_reclaim();
if (*pa != 0) {
bzero((void*)phystokv(*pa), size);
return KERN_SUCCESS;
}
}
if (m == VM_PAGE_NULL) {
return KERN_RESOURCE_SHORTAGE;
}
*pa = (pmap_paddr_t)ptoa(VM_PAGE_GET_PHYS_PAGE(m));
pmap_enqueue_pages(m);
OSAddAtomic(size >> PAGE_SHIFT, &inuse_pmap_pages_count);
OSAddAtomic64(size >> PAGE_SHIFT, &alloc_pmap_pages_count);
bzero((void*)phystokv(*pa), size);
return KERN_SUCCESS;
#endif
}
#if XNU_MONITOR
static pmap_paddr_t
pmap_alloc_page_for_kern(unsigned int options)
{
pmap_paddr_t paddr;
vm_page_t m;
while ((m = vm_page_grab()) == VM_PAGE_NULL) {
if (options & PMAP_PAGES_ALLOCATE_NOWAIT) {
return 0;
}
VM_PAGE_WAIT();
}
vm_page_lock_queues();
vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE);
vm_page_unlock_queues();
paddr = (pmap_paddr_t)ptoa(VM_PAGE_GET_PHYS_PAGE(m));
if (__improbable(paddr == 0)) {
panic("%s: paddr is 0", __func__);
}
pmap_enqueue_pages(m);
OSAddAtomic(1, &inuse_pmap_pages_count);
OSAddAtomic64(1, &alloc_pmap_pages_count);
return paddr;
}
static void
pmap_alloc_page_for_ppl(unsigned int options)
{
thread_t self = current_thread();
uint16_t thread_options = self->options;
self->options |= TH_OPT_VMPRIV;
pmap_paddr_t paddr = pmap_alloc_page_for_kern(options);
self->options = thread_options;
if (paddr != 0) {
pmap_mark_page_as_ppl_page(paddr);
}
}
static pmap_t
pmap_alloc_pmap(void)
{
pmap_t pmap = PMAP_NULL;
pmap_simple_lock(&pmap_free_list_lock);
if (pmap_free_list != PMAP_NULL) {
pmap = pmap_free_list;
pmap_free_list = *((pmap_t *)pmap);
if (!PMAP_PTR_IS_VALID(pmap)) {
panic("%s: allocated pmap is not valid, pmap=%p",
__FUNCTION__, pmap);
}
}
pmap_simple_unlock(&pmap_free_list_lock);
return pmap;
}
static void
pmap_free_pmap(pmap_t pmap)
{
if (!PMAP_PTR_IS_VALID(pmap)) {
panic("%s: pmap is not valid, "
"pmap=%p",
__FUNCTION__,
pmap);
}
pmap_simple_lock(&pmap_free_list_lock);
*((pmap_t *)pmap) = pmap_free_list;
pmap_free_list = pmap;
pmap_simple_unlock(&pmap_free_list_lock);
}
static void
pmap_bootstrap_pmap_free_list(void)
{
pmap_t cur_head = PMAP_NULL;
unsigned long i = 0;
simple_lock_init(&pmap_free_list_lock, 0);
for (i = 0; i < pmap_array_count; i++) {
*((pmap_t *)(&pmap_array[i])) = cur_head;
cur_head = &pmap_array[i];
}
pmap_free_list = cur_head;
}
#endif
static void
pmap_pages_free(
pmap_paddr_t pa,
unsigned size)
{
if (__improbable(pmap_pages_request_count != 0)) {
page_free_entry_t *page_entry;
pmap_simple_lock(&pmap_pages_lock);
if (pmap_pages_request_count != 0) {
pmap_pages_request_count--;
page_entry = (page_free_entry_t *)phystokv(pa);
page_entry->next = pmap_pages_reclaim_list;
pmap_pages_reclaim_list = page_entry;
pmap_simple_unlock(&pmap_pages_lock);
return;
}
pmap_simple_unlock(&pmap_pages_lock);
}
#if XNU_MONITOR
(void)size;
pmap_give_free_ppl_page(pa);
#else
vm_page_t m;
pmap_paddr_t pa_max;
OSAddAtomic(-(size >> PAGE_SHIFT), &inuse_pmap_pages_count);
for (pa_max = pa + size; pa < pa_max; pa = pa + PAGE_SIZE) {
vm_object_lock(pmap_object);
m = vm_page_lookup(pmap_object, (pa - gPhysBase));
assert(m != VM_PAGE_NULL);
assert(VM_PAGE_WIRED(m));
vm_page_lock_queues();
vm_page_free(m);
vm_page_unlock_queues();
vm_object_unlock(pmap_object);
}
#endif
}
static inline void
PMAP_ZINFO_PALLOC(
pmap_t pmap, int bytes)
{
pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes);
}
static inline void
PMAP_ZINFO_PFREE(
pmap_t pmap,
int bytes)
{
pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes);
}
static inline void
pmap_tt_ledger_credit(
pmap_t pmap,
vm_size_t size)
{
if (pmap != kernel_pmap) {
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, size);
pmap_ledger_credit(pmap, task_ledgers.page_table, size);
}
}
static inline void
pmap_tt_ledger_debit(
pmap_t pmap,
vm_size_t size)
{
if (pmap != kernel_pmap) {
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, size);
pmap_ledger_debit(pmap, task_ledgers.page_table, size);
}
}
static inline void
pmap_update_plru(uint16_t asid_index)
{
if (__probable(pmap_asid_plru)) {
unsigned plru_index = asid_index >> 6;
if (__improbable(os_atomic_andnot(&asid_plru_bitmap[plru_index], (1ULL << (asid_index & 63)), relaxed) == 0)) {
asid_plru_generation[plru_index] = ++asid_plru_gencount;
asid_plru_bitmap[plru_index] = ((plru_index == (MAX_HW_ASIDS >> 6)) ? ~(1ULL << 63) : UINT64_MAX);
}
}
}
static bool
alloc_asid(pmap_t pmap)
{
int vasid = -1;
uint16_t hw_asid;
pmap_simple_lock(&asid_lock);
if (__probable(pmap_asid_plru)) {
unsigned plru_index = 0;
uint64_t lowest_gen = asid_plru_generation[0];
uint64_t lowest_gen_bitmap = asid_plru_bitmap[0];
for (unsigned i = 1; i < (sizeof(asid_plru_generation) / sizeof(asid_plru_generation[0])); ++i) {
if (asid_plru_generation[i] < lowest_gen) {
plru_index = i;
lowest_gen = asid_plru_generation[i];
lowest_gen_bitmap = asid_plru_bitmap[i];
}
}
for (; plru_index < BITMAP_LEN(pmap_max_asids); plru_index += ((MAX_HW_ASIDS + 1) >> 6)) {
uint64_t temp_plru = lowest_gen_bitmap & asid_bitmap[plru_index];
if (temp_plru) {
vasid = (plru_index << 6) + lsb_first(temp_plru);
#if DEVELOPMENT || DEBUG
++pmap_asid_hits;
#endif
break;
}
}
}
if (__improbable(vasid < 0)) {
vasid = bitmap_lsb_first(&asid_bitmap[0], pmap_max_asids);
#if DEVELOPMENT || DEBUG
++pmap_asid_misses;
#endif
}
if (__improbable(vasid < 0)) {
pmap_simple_unlock(&asid_lock);
return false;
}
assert((uint32_t)vasid < pmap_max_asids);
assert(bitmap_test(&asid_bitmap[0], (unsigned int)vasid));
bitmap_clear(&asid_bitmap[0], (unsigned int)vasid);
pmap_simple_unlock(&asid_lock);
hw_asid = vasid % asid_chunk_size;
pmap->sw_asid = (uint8_t)(vasid / asid_chunk_size);
if (__improbable(hw_asid == MAX_HW_ASIDS)) {
assert(pmap->sw_asid < UINT8_MAX);
pmap->sw_asid = UINT8_MAX;
hw_asid = MAX_HW_ASIDS - 1 - (uint16_t)(vasid / asid_chunk_size);
assert(hw_asid < MAX_HW_ASIDS);
}
pmap_update_plru(hw_asid);
hw_asid += 1; #if __ARM_KERNEL_PROTECT__
hw_asid <<= 1; #endif
pmap->hw_asid = hw_asid;
return true;
}
static void
free_asid(pmap_t pmap)
{
unsigned int vasid;
uint16_t hw_asid = os_atomic_xchg(&pmap->hw_asid, 0, relaxed);
if (__improbable(hw_asid == 0)) {
return;
}
#if __ARM_KERNEL_PROTECT__
hw_asid >>= 1;
#endif
hw_asid -= 1;
if (__improbable(pmap->sw_asid == UINT8_MAX)) {
vasid = ((MAX_HW_ASIDS - 1 - hw_asid) * asid_chunk_size) + MAX_HW_ASIDS;
} else {
vasid = ((unsigned int)pmap->sw_asid * asid_chunk_size) + hw_asid;
}
if (__probable(pmap_asid_plru)) {
os_atomic_or(&asid_plru_bitmap[hw_asid >> 6], (1ULL << (hw_asid & 63)), relaxed);
}
pmap_simple_lock(&asid_lock);
assert(!bitmap_test(&asid_bitmap[0], vasid));
bitmap_set(&asid_bitmap[0], vasid);
pmap_simple_unlock(&asid_lock);
}
#if XNU_MONITOR
#define PV_LOW_WATER_MARK_DEFAULT (0x400)
#define PV_KERN_LOW_WATER_MARK_DEFAULT (0x400)
#define PV_ALLOC_CHUNK_INITIAL (0x400)
#define PV_KERN_ALLOC_CHUNK_INITIAL (0x400)
#define PV_CPU_MIN (0x80)
#define PV_CPU_MAX (0x400)
#else
#define PV_LOW_WATER_MARK_DEFAULT (0x200)
#define PV_KERN_LOW_WATER_MARK_DEFAULT (0x200)
#define PV_ALLOC_CHUNK_INITIAL (0x200)
#define PV_KERN_ALLOC_CHUNK_INITIAL (0x200)
#define PV_CPU_MIN (0x40)
#define PV_CPU_MAX (0x200)
#endif
#define PV_ALLOC_INITIAL_TARGET (PV_ALLOC_CHUNK_INITIAL * 5)
#define PV_KERN_ALLOC_INITIAL_TARGET (PV_KERN_ALLOC_CHUNK_INITIAL)
uint32_t pv_page_count MARK_AS_PMAP_DATA = 0;
uint32_t pv_kern_low_water_mark MARK_AS_PMAP_DATA = PV_KERN_LOW_WATER_MARK_DEFAULT;
uint32_t pv_alloc_initial_target MARK_AS_PMAP_DATA = PV_ALLOC_INITIAL_TARGET;
uint32_t pv_kern_alloc_initial_target MARK_AS_PMAP_DATA = PV_KERN_ALLOC_INITIAL_TARGET;
unsigned pmap_reserve_replenish_stat MARK_AS_PMAP_DATA;
unsigned pmap_kern_reserve_alloc_stat MARK_AS_PMAP_DATA;
static inline void pv_list_alloc(pv_entry_t **pv_ep);
static inline void pv_list_kern_alloc(pv_entry_t **pv_e);
static inline void pv_list_free(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt, uint32_t kern_target);
static pv_alloc_return_t
pv_alloc(
pmap_t pmap,
unsigned int pai,
pv_entry_t **pvepp)
{
if (pmap != NULL) {
pmap_assert_locked_w(pmap);
}
ASSERT_PVH_LOCKED(pai);
pv_list_alloc(pvepp);
if (PV_ENTRY_NULL != *pvepp) {
return PV_ALLOC_SUCCESS;
}
#if XNU_MONITOR
unsigned alloc_flags = PMAP_PAGES_ALLOCATE_NOWAIT;
#else
unsigned alloc_flags = 0;
#endif
if ((pmap == NULL) || (kernel_pmap == pmap)) {
pv_list_kern_alloc(pvepp);
if (PV_ENTRY_NULL != *pvepp) {
return PV_ALLOC_SUCCESS;
}
alloc_flags = PMAP_PAGES_ALLOCATE_NOWAIT | PMAP_PAGES_RECLAIM_NOWAIT;
}
pv_entry_t *pv_e;
pv_entry_t *pv_eh;
pv_entry_t *pv_et;
int pv_cnt;
pmap_paddr_t pa;
kern_return_t ret;
pv_alloc_return_t pv_status = PV_ALLOC_RETRY;
UNLOCK_PVH(pai);
if (pmap != NULL) {
pmap_unlock(pmap);
}
ret = pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, alloc_flags);
if (ret != KERN_SUCCESS) {
pv_status = PV_ALLOC_FAIL;
goto pv_alloc_cleanup;
}
pv_page_count++;
pv_e = (pv_entry_t *)phystokv(pa);
*pvepp = pv_e;
pv_cnt = (PAGE_SIZE / sizeof(pv_entry_t)) - 1;
pv_eh = pv_e + 1;
pv_et = &pv_e[pv_cnt];
pv_list_free(pv_eh, pv_et, pv_cnt, pv_kern_low_water_mark);
pv_alloc_cleanup:
if (pmap != NULL) {
pmap_lock(pmap);
}
LOCK_PVH(pai);
return pv_status;
}
static inline void
pv_free_entry(
pv_entry_t *pvep)
{
pv_list_free(pvep, pvep, 1, pv_kern_low_water_mark);
}
static inline void
pv_free_list_alloc(pv_free_list_t *free_list, pv_entry_t **pv_ep)
{
assert(((free_list->list != NULL) && (free_list->count > 0)) ||
((free_list->list == NULL) && (free_list->count == 0)));
if ((*pv_ep = free_list->list) != NULL) {
pv_entry_t *pv_e = *pv_ep;
if ((pv_e->pve_next == NULL) && (free_list->count > 1)) {
free_list->list = pv_e + 1;
} else {
free_list->list = pv_e->pve_next;
pv_e->pve_next = PV_ENTRY_NULL;
}
free_list->count--;
}
}
static inline void
pv_list_alloc(pv_entry_t **pv_ep)
{
assert(*pv_ep == PV_ENTRY_NULL);
#if !XNU_MONITOR
mp_disable_preemption();
#endif
pmap_cpu_data_t *pmap_cpu_data = pmap_get_cpu_data();
pv_free_list_alloc(&pmap_cpu_data->pv_free, pv_ep);
#if !XNU_MONITOR
mp_enable_preemption();
#endif
if (*pv_ep != PV_ENTRY_NULL) {
return;
}
#if !XNU_MONITOR
if (pv_kern_free.count < pv_kern_low_water_mark) {
return;
}
#endif
pmap_simple_lock(&pv_free_list_lock);
pv_free_list_alloc(&pv_free, pv_ep);
pmap_simple_unlock(&pv_free_list_lock);
}
static inline void
pv_list_free(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt, uint32_t kern_target)
{
if (pv_cnt == 1) {
bool limit_exceeded = false;
#if !XNU_MONITOR
mp_disable_preemption();
#endif
pmap_cpu_data_t *pmap_cpu_data = pmap_get_cpu_data();
pv_et->pve_next = pmap_cpu_data->pv_free.list;
pmap_cpu_data->pv_free.list = pv_eh;
if (pmap_cpu_data->pv_free.count == PV_CPU_MIN) {
pmap_cpu_data->pv_free_tail = pv_et;
}
pmap_cpu_data->pv_free.count += pv_cnt;
if (__improbable(pmap_cpu_data->pv_free.count > PV_CPU_MAX)) {
pv_et = pmap_cpu_data->pv_free_tail;
pv_cnt = pmap_cpu_data->pv_free.count - PV_CPU_MIN;
pmap_cpu_data->pv_free.list = pmap_cpu_data->pv_free_tail->pve_next;
pmap_cpu_data->pv_free.count = PV_CPU_MIN;
limit_exceeded = true;
}
#if !XNU_MONITOR
mp_enable_preemption();
#endif
if (__probable(!limit_exceeded)) {
return;
}
}
if (__improbable(pv_kern_free.count < kern_target)) {
pmap_simple_lock(&pv_kern_free_list_lock);
pv_et->pve_next = pv_kern_free.list;
pv_kern_free.list = pv_eh;
pv_kern_free.count += pv_cnt;
pmap_simple_unlock(&pv_kern_free_list_lock);
} else {
pmap_simple_lock(&pv_free_list_lock);
pv_et->pve_next = pv_free.list;
pv_free.list = pv_eh;
pv_free.count += pv_cnt;
pmap_simple_unlock(&pv_free_list_lock);
}
}
static inline void
pv_list_kern_alloc(pv_entry_t **pv_ep)
{
assert(*pv_ep == PV_ENTRY_NULL);
pmap_simple_lock(&pv_kern_free_list_lock);
if (pv_kern_free.count > 0) {
pmap_kern_reserve_alloc_stat++;
}
pv_free_list_alloc(&pv_kern_free, pv_ep);
pmap_simple_unlock(&pv_kern_free_list_lock);
}
void
mapping_adjust(void)
{
}
MARK_AS_PMAP_TEXT static kern_return_t
mapping_replenish_internal(uint32_t kern_target_count, uint32_t user_target_count)
{
pv_entry_t *pv_eh;
pv_entry_t *pv_et;
int pv_cnt;
pmap_paddr_t pa;
kern_return_t ret = KERN_SUCCESS;
while ((pv_free.count < user_target_count) || (pv_kern_free.count < kern_target_count)) {
#if XNU_MONITOR
if ((ret = pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT)) != KERN_SUCCESS) {
return ret;
}
#else
ret = pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, 0);
assert(ret == KERN_SUCCESS);
#endif
pv_page_count++;
pv_eh = (pv_entry_t *)phystokv(pa);
pv_cnt = PAGE_SIZE / sizeof(pv_entry_t);
pv_et = &pv_eh[pv_cnt - 1];
pmap_reserve_replenish_stat += pv_cnt;
pv_list_free(pv_eh, pv_et, pv_cnt, kern_target_count);
}
return ret;
}
MARK_AS_PMAP_TEXT static kern_return_t
mapping_free_prime_internal(void)
{
return mapping_replenish_internal(pv_kern_alloc_initial_target, pv_alloc_initial_target);
}
void
mapping_free_prime(void)
{
kern_return_t kr = KERN_FAILURE;
#if XNU_MONITOR
unsigned int i = 0;
for (i = 0; i < pv_alloc_initial_target; i += (PAGE_SIZE / sizeof(pv_entry_t))) {
pmap_alloc_page_for_ppl(0);
}
for (i = 0; i < pv_kern_alloc_initial_target; i += (PAGE_SIZE / sizeof(pv_entry_t))) {
pmap_alloc_page_for_ppl(0);
}
while ((kr = mapping_free_prime_ppl()) == KERN_RESOURCE_SHORTAGE) {
pmap_alloc_page_for_ppl(0);
}
#else
kr = mapping_free_prime_internal();
#endif
if (kr != KERN_SUCCESS) {
panic("%s: failed, kr=%d",
__FUNCTION__, kr);
}
}
static void
ptd_bootstrap(
pt_desc_t *ptdp,
unsigned int ptd_cnt)
{
simple_lock_init(&ptd_free_list_lock, 0);
*((void**)(&ptdp[ptd_cnt - 1])) = (void*)ptd_free_list;
ptd_free_list = ptdp;
ptd_free_count += ptd_cnt;
ptd_preboot = FALSE;
}
static pt_desc_t*
ptd_alloc_unlinked(void)
{
pt_desc_t *ptdp;
unsigned i;
if (!ptd_preboot) {
pmap_simple_lock(&ptd_free_list_lock);
}
assert(((ptd_free_list != NULL) && (ptd_free_count > 0)) ||
((ptd_free_list == NULL) && (ptd_free_count == 0)));
if (ptd_free_count == 0) {
unsigned int ptd_cnt = PAGE_SIZE / sizeof(pt_desc_t);
if (ptd_preboot) {
ptdp = (pt_desc_t *)avail_start;
avail_start += PAGE_SIZE;
bzero(ptdp, PAGE_SIZE);
} else {
pmap_paddr_t pa;
pmap_simple_unlock(&ptd_free_list_lock);
if (pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT) != KERN_SUCCESS) {
return NULL;
}
ptdp = (pt_desc_t *)phystokv(pa);
pmap_simple_lock(&ptd_free_list_lock);
}
*((void**)(&ptdp[ptd_cnt - 1])) = (void*)ptd_free_list;
ptd_free_list = ptdp;
ptd_free_count += ptd_cnt;
}
if ((ptdp = ptd_free_list) != PTD_ENTRY_NULL) {
ptd_free_list = (pt_desc_t *)(*(void **)ptdp);
if ((ptd_free_list == NULL) && (ptd_free_count > 1)) {
ptd_free_list = ptdp + 1;
}
ptd_free_count--;
} else {
panic("%s: out of ptd entry",
__FUNCTION__);
}
if (!ptd_preboot) {
pmap_simple_unlock(&ptd_free_list_lock);
}
ptdp->pt_page.next = NULL;
ptdp->pt_page.prev = NULL;
ptdp->pmap = NULL;
for (i = 0; i < PT_INDEX_MAX; i++) {
ptdp->ptd_info[i].va = (vm_offset_t)-1;
ptdp->ptd_info[i].refcnt = 0;
ptdp->ptd_info[i].wiredcnt = 0;
}
return ptdp;
}
static inline pt_desc_t*
ptd_alloc(pmap_t pmap)
{
pt_desc_t *ptdp = ptd_alloc_unlinked();
if (ptdp == NULL) {
return NULL;
}
ptdp->pmap = pmap;
if (pmap != kernel_pmap) {
pmap_simple_lock(&pt_pages_lock);
queue_enter(&pt_page_list, ptdp, pt_desc_t *, pt_page);
pmap_simple_unlock(&pt_pages_lock);
}
pmap_tt_ledger_credit(pmap, sizeof(*ptdp));
return ptdp;
}
static void
ptd_deallocate(pt_desc_t *ptdp)
{
pmap_t pmap = ptdp->pmap;
if (ptd_preboot) {
panic("%s: early boot, "
"ptdp=%p",
__FUNCTION__,
ptdp);
}
if (ptdp->pt_page.next != NULL) {
pmap_simple_lock(&pt_pages_lock);
queue_remove(&pt_page_list, ptdp, pt_desc_t *, pt_page);
pmap_simple_unlock(&pt_pages_lock);
}
pmap_simple_lock(&ptd_free_list_lock);
(*(void **)ptdp) = (void *)ptd_free_list;
ptd_free_list = (pt_desc_t *)ptdp;
ptd_free_count++;
pmap_simple_unlock(&ptd_free_list_lock);
if (pmap != NULL) {
pmap_tt_ledger_debit(pmap, sizeof(*ptdp));
}
}
static void
ptd_init(
pt_desc_t *ptdp,
pmap_t pmap,
vm_map_address_t va,
unsigned int level,
pt_entry_t *pte_p)
{
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if (ptdp->pmap != pmap) {
panic("%s: pmap mismatch, "
"ptdp=%p, pmap=%p, va=%p, level=%u, pte_p=%p",
__FUNCTION__,
ptdp, pmap, (void*)va, level, pte_p);
}
assert(level > pt_attr_root_level(pt_attr));
ptd_info_t *ptd_info = ptd_get_info(ptdp, pte_p);
ptd_info->va = (vm_offset_t) va & ~pt_attr_ln_pt_offmask(pt_attr, level - 1);
if (level < pt_attr_leaf_level(pt_attr)) {
ptd_info->refcnt = PT_DESC_REFCOUNT;
}
}
boolean_t
pmap_valid_address(
pmap_paddr_t addr)
{
return pa_valid(addr);
}
#if (__ARM_VMSA__ == 7)
static inline tt_entry_t *
pmap_tte(pmap_t pmap,
vm_map_address_t addr)
{
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if (!(tte_index(pmap, pt_attr, addr) < pmap->tte_index_max)) {
return (tt_entry_t *)NULL;
}
return &pmap->tte[tte_index(pmap, pt_attr, addr)];
}
static inline pt_entry_t *
pmap_pte(
pmap_t pmap,
vm_map_address_t addr)
{
pt_entry_t *ptp;
tt_entry_t *ttp;
tt_entry_t tte;
ttp = pmap_tte(pmap, addr);
if (ttp == (tt_entry_t *)NULL) {
return PT_ENTRY_NULL;
}
tte = *ttp;
#if MACH_ASSERT
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
panic("%s: Attempt to demote L1 block, tte=0x%lx, "
"pmap=%p, addr=%p",
__FUNCTION__, (unsigned long)tte,
pmap, (void*)addr);
}
#endif
if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) {
return PT_ENTRY_NULL;
}
ptp = (pt_entry_t *) ttetokv(tte) + pte_index(pmap, pt_addr, addr);
return ptp;
}
__unused static inline tt_entry_t *
pmap_ttne(pmap_t pmap,
unsigned int target_level,
vm_map_address_t addr)
{
tt_entry_t * ret_ttep = NULL;
switch (target_level) {
case 1:
ret_ttep = pmap_tte(pmap, addr);
break;
case 2:
ret_ttep = (tt_entry_t *)pmap_pte(pmap, addr);
break;
default:
panic("%s: bad level, "
"pmap=%p, target_level=%u, addr=%p",
__FUNCTION__,
pmap, target_level, (void *)addr);
}
return ret_ttep;
}
#else
static inline tt_entry_t *
pmap_ttne(pmap_t pmap,
unsigned int target_level,
vm_map_address_t addr)
{
tt_entry_t * ttp = NULL;
tt_entry_t * ttep = NULL;
tt_entry_t tte = ARM_TTE_EMPTY;
unsigned int cur_level;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
ttp = pmap->tte;
assert(target_level <= pt_attr->pta_max_level);
for (cur_level = pt_attr->pta_root_level; cur_level <= target_level; cur_level++) {
ttep = &ttp[ttn_index(pmap, pt_attr, addr, cur_level)];
if (cur_level == target_level) {
break;
}
tte = *ttep;
#if MACH_ASSERT
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) {
panic("%s: Attempt to demote L%u block, tte=0x%llx, "
"pmap=%p, target_level=%u, addr=%p",
__FUNCTION__, cur_level, tte,
pmap, target_level, (void*)addr);
}
#endif
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
return TT_ENTRY_NULL;
}
ttp = (tt_entry_t*)phystokv(tte & ARM_TTE_TABLE_MASK);
}
return ttep;
}
static inline tt_entry_t *
pmap_tt1e(pmap_t pmap,
vm_map_address_t addr)
{
return pmap_ttne(pmap, PMAP_TT_L1_LEVEL, addr);
}
static inline tt_entry_t *
pmap_tt2e(pmap_t pmap,
vm_map_address_t addr)
{
return pmap_ttne(pmap, PMAP_TT_L2_LEVEL, addr);
}
static inline pt_entry_t *
pmap_tt3e(
pmap_t pmap,
vm_map_address_t addr)
{
return (pt_entry_t*)pmap_ttne(pmap, PMAP_TT_L3_LEVEL, addr);
}
static inline tt_entry_t *
pmap_tte(
pmap_t pmap,
vm_map_address_t addr)
{
return pmap_tt2e(pmap, addr);
}
static inline pt_entry_t *
pmap_pte(
pmap_t pmap,
vm_map_address_t addr)
{
return pmap_tt3e(pmap, addr);
}
#endif
vm_map_address_t
pmap_map(
vm_map_address_t virt,
vm_offset_t start,
vm_offset_t end,
vm_prot_t prot,
unsigned int flags)
{
kern_return_t kr;
vm_size_t ps;
ps = PAGE_SIZE;
while (start < end) {
kr = pmap_enter(kernel_pmap, virt, (ppnum_t)atop(start),
prot, VM_PROT_NONE, flags, FALSE);
if (kr != KERN_SUCCESS) {
panic("%s: failed pmap_enter, "
"virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x",
__FUNCTION__,
(void *) virt, (void *) start, (void *) end, prot, flags);
}
virt += ps;
start += ps;
}
return virt;
}
vm_map_address_t
pmap_map_bd_with_options(
vm_map_address_t virt,
vm_offset_t start,
vm_offset_t end,
vm_prot_t prot,
int32_t options)
{
pt_entry_t tmplate;
pt_entry_t *ptep;
vm_map_address_t vaddr;
vm_offset_t paddr;
pt_entry_t mem_attr;
switch (options & PMAP_MAP_BD_MASK) {
case PMAP_MAP_BD_WCOMB:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB);
#if (__ARM_VMSA__ > 7)
mem_attr |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
mem_attr |= ARM_PTE_SH;
#endif
break;
case PMAP_MAP_BD_POSTED:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED);
break;
case PMAP_MAP_BD_POSTED_REORDERED:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED_REORDERED);
break;
case PMAP_MAP_BD_POSTED_COMBINED_REORDERED:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED_COMBINED_REORDERED);
break;
default:
mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
break;
}
tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) |
mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF;
#if __ARM_KERNEL_PROTECT__
tmplate |= ARM_PTE_NG;
#endif
vaddr = virt;
paddr = start;
while (paddr < end) {
ptep = pmap_pte(kernel_pmap, vaddr);
if (ptep == PT_ENTRY_NULL) {
panic("%s: no PTE for vaddr=%p, "
"virt=%p, start=%p, end=%p, prot=0x%x, options=0x%x",
__FUNCTION__, (void*)vaddr,
(void*)virt, (void*)start, (void*)end, prot, options);
}
assert(!ARM_PTE_IS_COMPRESSED(*ptep, ptep));
WRITE_PTE_STRONG(ptep, tmplate);
pte_increment_pa(tmplate);
vaddr += PAGE_SIZE;
paddr += PAGE_SIZE;
}
if (end >= start) {
flush_mmu_tlb_region(virt, (unsigned)(end - start));
}
return vaddr;
}
vm_map_address_t
pmap_map_bd(
vm_map_address_t virt,
vm_offset_t start,
vm_offset_t end,
vm_prot_t prot)
{
pt_entry_t tmplate;
pt_entry_t *ptep;
vm_map_address_t vaddr;
vm_offset_t paddr;
tmplate = pa_to_pte(start)
| ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
| ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
#if __ARM_KERNEL_PROTECT__
tmplate |= ARM_PTE_NG;
#endif
vaddr = virt;
paddr = start;
while (paddr < end) {
ptep = pmap_pte(kernel_pmap, vaddr);
if (ptep == PT_ENTRY_NULL) {
panic("pmap_map_bd");
}
assert(!ARM_PTE_IS_COMPRESSED(*ptep, ptep));
WRITE_PTE_STRONG(ptep, tmplate);
pte_increment_pa(tmplate);
vaddr += PAGE_SIZE;
paddr += PAGE_SIZE;
}
if (end >= start) {
flush_mmu_tlb_region(virt, (unsigned)(end - start));
}
return vaddr;
}
vm_map_address_t
pmap_map_high_window_bd(
vm_offset_t pa_start,
vm_size_t len,
vm_prot_t prot)
{
pt_entry_t *ptep, pte;
#if (__ARM_VMSA__ == 7)
vm_map_address_t va_start = VM_HIGH_KERNEL_WINDOW;
vm_map_address_t va_max = VM_MAX_KERNEL_ADDRESS;
#else
vm_map_address_t va_start = VREGION1_START;
vm_map_address_t va_max = VREGION1_START + VREGION1_SIZE;
#endif
vm_map_address_t va_end;
vm_map_address_t va;
vm_size_t offset;
offset = pa_start & PAGE_MASK;
pa_start -= offset;
len += offset;
if (len > (va_max - va_start)) {
panic("%s: area too large, "
"pa_start=%p, len=%p, prot=0x%x",
__FUNCTION__,
(void*)pa_start, (void*)len, prot);
}
scan:
for (; va_start < va_max; va_start += PAGE_SIZE) {
ptep = pmap_pte(kernel_pmap, va_start);
assert(!ARM_PTE_IS_COMPRESSED(*ptep, ptep));
if (*ptep == ARM_PTE_TYPE_FAULT) {
break;
}
}
if (va_start > va_max) {
panic("%s: insufficient pages, "
"pa_start=%p, len=%p, prot=0x%x",
__FUNCTION__,
(void*)pa_start, (void*)len, prot);
}
for (va_end = va_start + PAGE_SIZE; va_end < va_start + len; va_end += PAGE_SIZE) {
ptep = pmap_pte(kernel_pmap, va_end);
assert(!ARM_PTE_IS_COMPRESSED(*ptep, ptep));
if (*ptep != ARM_PTE_TYPE_FAULT) {
va_start = va_end + PAGE_SIZE;
goto scan;
}
}
for (va = va_start; va < va_end; va += PAGE_SIZE, pa_start += PAGE_SIZE) {
ptep = pmap_pte(kernel_pmap, va);
pte = pa_to_pte(pa_start)
| ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
| ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
WRITE_PTE_STRONG(ptep, pte);
}
PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len, false);
#if KASAN
kasan_notify_address(va_start, len);
#endif
return va_start;
}
#define PMAP_ALIGN(addr, align) ((addr) + ((align) - 1) & ~((align) - 1))
static void
pmap_compute_pv_targets(void)
{
DTEntry entry;
void const *prop = NULL;
int err;
unsigned int prop_size;
err = SecureDTLookupEntry(NULL, "/defaults", &entry);
assert(err == kSuccess);
if (kSuccess == SecureDTGetProperty(entry, "pmap-pv-count", &prop, &prop_size)) {
if (prop_size != sizeof(pv_alloc_initial_target)) {
panic("pmap-pv-count property is not a 32-bit integer");
}
pv_alloc_initial_target = *((uint32_t const *)prop);
}
if (kSuccess == SecureDTGetProperty(entry, "pmap-kern-pv-count", &prop, &prop_size)) {
if (prop_size != sizeof(pv_kern_alloc_initial_target)) {
panic("pmap-kern-pv-count property is not a 32-bit integer");
}
pv_kern_alloc_initial_target = *((uint32_t const *)prop);
}
if (kSuccess == SecureDTGetProperty(entry, "pmap-kern-pv-min", &prop, &prop_size)) {
if (prop_size != sizeof(pv_kern_low_water_mark)) {
panic("pmap-kern-pv-min property is not a 32-bit integer");
}
pv_kern_low_water_mark = *((uint32_t const *)prop);
}
}
static uint32_t
pmap_compute_max_asids(void)
{
DTEntry entry;
void const *prop = NULL;
uint32_t max_asids;
int err;
unsigned int prop_size;
err = SecureDTLookupEntry(NULL, "/defaults", &entry);
assert(err == kSuccess);
if (kSuccess != SecureDTGetProperty(entry, "pmap-max-asids", &prop, &prop_size)) {
return MAX_ASIDS;
}
if (prop_size != sizeof(max_asids)) {
panic("pmap-max-asids property is not a 32-bit integer");
}
max_asids = *((uint32_t const *)prop);
max_asids = (max_asids + 63) & ~63UL;
if (((max_asids + MAX_HW_ASIDS) / (MAX_HW_ASIDS + 1)) > MIN(MAX_HW_ASIDS, UINT8_MAX)) {
panic("pmap-max-asids too large");
}
if (max_asids == 0) {
panic("pmap-max-asids cannot be zero");
}
return max_asids;
}
static vm_size_t
pmap_compute_io_rgns(void)
{
DTEntry entry;
pmap_io_range_t const *ranges;
uint64_t rgn_end;
void const *prop = NULL;
int err;
unsigned int prop_size;
err = SecureDTLookupEntry(NULL, "/defaults", &entry);
assert(err == kSuccess);
if (kSuccess != SecureDTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size)) {
return 0;
}
ranges = prop;
for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) {
if (ranges[i].addr & PAGE_MASK) {
panic("pmap I/O region %u addr 0x%llx is not page-aligned", i, ranges[i].addr);
}
if (ranges[i].len & PAGE_MASK) {
panic("pmap I/O region %u length 0x%llx is not page-aligned", i, ranges[i].len);
}
if (os_add_overflow(ranges[i].addr, ranges[i].len, &rgn_end)) {
panic("pmap I/O region %u addr 0x%llx length 0x%llx wraps around", i, ranges[i].addr, ranges[i].len);
}
if (((ranges[i].addr <= gPhysBase) && (rgn_end > gPhysBase)) ||
((ranges[i].addr < avail_end) && (rgn_end >= avail_end)) ||
((ranges[i].addr > gPhysBase) && (rgn_end < avail_end))) {
panic("pmap I/O region %u addr 0x%llx length 0x%llx overlaps physical memory", i, ranges[i].addr, ranges[i].len);
}
++num_io_rgns;
}
return num_io_rgns * sizeof(*ranges);
}
typedef int (*cmpfunc_t)(const void *a, const void *b);
extern void
qsort(void *a, size_t n, size_t es, cmpfunc_t cmp);
static int
cmp_io_rgns(const void *a, const void *b)
{
const pmap_io_range_t *range_a = a;
const pmap_io_range_t *range_b = b;
if ((range_b->addr + range_b->len) <= range_a->addr) {
return 1;
} else if ((range_a->addr + range_a->len) <= range_b->addr) {
return -1;
} else {
return 0;
}
}
static void
pmap_load_io_rgns(void)
{
DTEntry entry;
pmap_io_range_t const *ranges;
void const *prop = NULL;
int err;
unsigned int prop_size;
if (num_io_rgns == 0) {
return;
}
err = SecureDTLookupEntry(NULL, "/defaults", &entry);
assert(err == kSuccess);
err = SecureDTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size);
assert(err == kSuccess);
ranges = prop;
for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) {
io_attr_table[i] = ranges[i];
}
qsort(io_attr_table, num_io_rgns, sizeof(*ranges), cmp_io_rgns);
}
#if __arm64__
uint64_t
pmap_get_arm64_prot(
pmap_t pmap,
vm_offset_t addr)
{
tt_entry_t tte = 0;
unsigned int level = 0;
uint64_t tte_type = 0;
uint64_t effective_prot_bits = 0;
uint64_t aggregate_tte = 0;
uint64_t table_ap_bits = 0, table_xn = 0, table_pxn = 0;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
for (level = pt_attr->pta_root_level; level <= pt_attr->pta_max_level; level++) {
tte = *pmap_ttne(pmap, level, addr);
if (!(tte & ARM_TTE_VALID)) {
return 0;
}
tte_type = tte & ARM_TTE_TYPE_MASK;
if ((tte_type == ARM_TTE_TYPE_BLOCK) ||
(level == pt_attr->pta_max_level)) {
break;
} else if (tte_type == ARM_TTE_TYPE_TABLE) {
aggregate_tte |= tte;
}
}
table_ap_bits = ((aggregate_tte >> ARM_TTE_TABLE_APSHIFT) & AP_MASK);
table_xn = (aggregate_tte & ARM_TTE_TABLE_XN);
table_pxn = (aggregate_tte & ARM_TTE_TABLE_PXN);
effective_prot_bits = tte & (ARM_PTE_APMASK | ARM_PTE_NX | ARM_PTE_PNX);
effective_prot_bits &= ~(ARM_PTE_AP(table_ap_bits));
effective_prot_bits |= (table_xn ? ARM_PTE_NX : 0);
effective_prot_bits |= (table_pxn ? ARM_PTE_PNX : 0);
return effective_prot_bits;
}
#endif
void
pmap_bootstrap(
vm_offset_t vstart)
{
pmap_paddr_t pmap_struct_start;
vm_size_t pv_head_size;
vm_size_t ptd_root_table_size;
vm_size_t pp_attr_table_size;
vm_size_t io_attr_table_size;
vm_size_t asid_table_size;
unsigned int npages;
vm_map_offset_t maxoffset;
lck_grp_init(&pmap_lck_grp, "pmap", LCK_GRP_ATTR_NULL);
#if XNU_MONITOR
#if DEVELOPMENT || DEBUG
PE_parse_boot_argn("-unsafe_kernel_text", &pmap_ppl_disable, sizeof(pmap_ppl_disable));
#endif
#if CONFIG_CSR_FROM_DT
if (csr_unsafe_kernel_text) {
pmap_ppl_disable = true;
}
#endif
#endif
#if DEVELOPMENT || DEBUG
if (PE_parse_boot_argn("pmap_trace", &pmap_trace_mask, sizeof(pmap_trace_mask))) {
kprintf("Kernel traces for pmap operations enabled\n");
}
#endif
pmap_stamp = 1;
#if ARM_PARAMETERIZED_PMAP
kernel_pmap->pmap_pt_attr = native_pt_attr;
#endif
#if HAS_APPLE_PAC
kernel_pmap->disable_jop = 0;
#endif
kernel_pmap->tte = cpu_tte;
kernel_pmap->ttep = cpu_ttep;
#if (__ARM_VMSA__ > 7)
kernel_pmap->min = ARM64_TTBR1_MIN_ADDR;
#else
kernel_pmap->min = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
#endif
kernel_pmap->max = VM_MAX_KERNEL_ADDRESS;
os_atomic_init(&kernel_pmap->ref_count, 1);
kernel_pmap->gc_status = 0;
kernel_pmap->nx_enabled = TRUE;
#ifdef __arm64__
kernel_pmap->is_64bit = TRUE;
#else
kernel_pmap->is_64bit = FALSE;
#endif
kernel_pmap->stamp = os_atomic_inc(&pmap_stamp, relaxed);
#if ARM_PARAMETERIZED_PMAP
kernel_pmap->pmap_pt_attr = native_pt_attr;
#endif
kernel_pmap->nested_region_addr = 0x0ULL;
kernel_pmap->nested_region_size = 0x0ULL;
kernel_pmap->nested_region_asid_bitmap = NULL;
kernel_pmap->nested_region_asid_bitmap_size = 0x0UL;
#if (__ARM_VMSA__ == 7)
kernel_pmap->tte_index_max = 4 * NTTES;
#endif
kernel_pmap->hw_asid = 0;
kernel_pmap->sw_asid = 0;
pmap_lock_init(kernel_pmap);
memset((void *) &kernel_pmap->stats, 0, sizeof(kernel_pmap->stats));
io_attr_table_size = pmap_compute_io_rgns();
npages = (unsigned int)atop(mem_size);
pp_attr_table_size = npages * sizeof(pp_attr_t);
pv_head_size = round_page(sizeof(pv_entry_t *) * npages);
ptd_root_table_size = sizeof(pt_desc_t) * (mem_size / ((PAGE_SIZE / sizeof(pt_entry_t)) * ARM_PGBYTES)) * 2;
pmap_max_asids = pmap_compute_max_asids();
pmap_asid_plru = (pmap_max_asids > MAX_HW_ASIDS);
PE_parse_boot_argn("pmap_asid_plru", &pmap_asid_plru, sizeof(pmap_asid_plru));
_Static_assert(sizeof(asid_plru_bitmap[0] == sizeof(uint64_t)), "bitmap_t is not a 64-bit integer");
_Static_assert(((MAX_HW_ASIDS + 1) % 64) == 0, "MAX_HW_ASIDS + 1 is not divisible by 64");
asid_chunk_size = (pmap_asid_plru ? (MAX_HW_ASIDS + 1) : MAX_HW_ASIDS);
asid_table_size = sizeof(*asid_bitmap) * BITMAP_LEN(pmap_max_asids);
pmap_compute_pv_targets();
pmap_struct_start = avail_start;
pp_attr_table = (pp_attr_t *) phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + pp_attr_table_size, __alignof(pp_attr_t));
io_attr_table = (pmap_io_range_t *) phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + io_attr_table_size, __alignof(pv_entry_t*));
pv_head_table = (pv_entry_t **) phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + pv_head_size, __alignof(pt_desc_t));
ptd_root_table = (pt_desc_t *)phystokv(avail_start);
avail_start = PMAP_ALIGN(avail_start + ptd_root_table_size, __alignof(bitmap_t));
asid_bitmap = (bitmap_t*)phystokv(avail_start);
avail_start = round_page(avail_start + asid_table_size);
memset((char *)phystokv(pmap_struct_start), 0, avail_start - pmap_struct_start);
pmap_load_io_rgns();
ptd_bootstrap(ptd_root_table, (unsigned int)(ptd_root_table_size / sizeof(pt_desc_t)));
#if XNU_MONITOR
pmap_array_begin = (void *)phystokv(avail_start);
pmap_array = pmap_array_begin;
avail_start += round_page(PMAP_ARRAY_SIZE * sizeof(struct pmap));
pmap_array_end = (void *)phystokv(avail_start);
pmap_array_count = ((pmap_array_end - pmap_array_begin) / sizeof(struct pmap));
pmap_bootstrap_pmap_free_list();
pmap_ledger_ptr_array_begin = (void *)phystokv(avail_start);
pmap_ledger_ptr_array = pmap_ledger_ptr_array_begin;
avail_start += round_page(MAX_PMAP_LEDGERS * sizeof(void*));
pmap_ledger_ptr_array_end = (void *)phystokv(avail_start);
pmap_ledger_refcnt_begin = (void *)phystokv(avail_start);
pmap_ledger_refcnt = pmap_ledger_refcnt_begin;
avail_start += round_page(MAX_PMAP_LEDGERS * sizeof(os_refcnt_t));
pmap_ledger_refcnt_end = (void *)phystokv(avail_start);
#endif
pmap_cpu_data_array_init();
vm_first_phys = gPhysBase;
vm_last_phys = trunc_page(avail_end);
queue_init(&map_pmap_list);
queue_enter(&map_pmap_list, kernel_pmap, pmap_t, pmaps);
free_page_size_tt_list = TT_FREE_ENTRY_NULL;
free_page_size_tt_count = 0;
free_page_size_tt_max = 0;
free_two_page_size_tt_list = TT_FREE_ENTRY_NULL;
free_two_page_size_tt_count = 0;
free_two_page_size_tt_max = 0;
free_tt_list = TT_FREE_ENTRY_NULL;
free_tt_count = 0;
free_tt_max = 0;
queue_init(&pt_page_list);
pmap_pages_request_count = 0;
pmap_pages_request_acum = 0;
pmap_pages_reclaim_list = PAGE_FREE_ENTRY_NULL;
virtual_space_start = vstart;
virtual_space_end = VM_MAX_KERNEL_ADDRESS;
bitmap_full(&asid_bitmap[0], pmap_max_asids);
bitmap_full(&asid_plru_bitmap[0], MAX_HW_ASIDS);
asid_plru_bitmap[MAX_HW_ASIDS >> 6] = ~(1ULL << 63);
if (PE_parse_boot_argn("arm_maxoffset", &maxoffset, sizeof(maxoffset))) {
maxoffset = trunc_page(maxoffset);
if ((maxoffset >= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MIN))
&& (maxoffset <= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MAX))) {
arm_pmap_max_offset_default = maxoffset;
}
}
#if defined(__arm64__)
if (PE_parse_boot_argn("arm64_maxoffset", &maxoffset, sizeof(maxoffset))) {
maxoffset = trunc_page(maxoffset);
if ((maxoffset >= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MIN))
&& (maxoffset <= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MAX))) {
arm64_pmap_max_offset_default = maxoffset;
}
}
#endif
PE_parse_boot_argn("pmap_panic_dev_wimg_on_managed", &pmap_panic_dev_wimg_on_managed, sizeof(pmap_panic_dev_wimg_on_managed));
#if MACH_ASSERT
PE_parse_boot_argn("pmap_stats_assert",
&pmap_stats_assert,
sizeof(pmap_stats_assert));
PE_parse_boot_argn("vm_footprint_suspend_allowed",
&vm_footprint_suspend_allowed,
sizeof(vm_footprint_suspend_allowed));
#endif
#if KASAN
kasan_map_shadow(CPUWINDOWS_BASE, CPUWINDOWS_TOP - CPUWINDOWS_BASE, true);
#endif
}
#if XNU_MONITOR
static inline void
pa_set_range_monitor(pmap_paddr_t start_pa, pmap_paddr_t end_pa)
{
pmap_paddr_t cur_pa;
for (cur_pa = start_pa; cur_pa < end_pa; cur_pa += ARM_PGBYTES) {
assert(pa_valid(cur_pa));
pa_set_monitor(cur_pa);
}
}
static void
pa_set_range_xprr_perm(pmap_paddr_t start_pa,
pmap_paddr_t end_pa,
unsigned int expected_perm,
unsigned int new_perm)
{
vm_offset_t start_va = phystokv(start_pa);
vm_offset_t end_va = start_va + (end_pa - start_pa);
pa_set_range_monitor(start_pa, end_pa);
pmap_set_range_xprr_perm(start_va, end_va, expected_perm, new_perm);
}
static void
pmap_lockdown_kc(void)
{
extern vm_offset_t vm_kernelcache_base;
extern vm_offset_t vm_kernelcache_top;
pmap_paddr_t start_pa = kvtophys(vm_kernelcache_base);
pmap_paddr_t end_pa = start_pa + (vm_kernelcache_top - vm_kernelcache_base);
pmap_paddr_t cur_pa = start_pa;
vm_offset_t cur_va = vm_kernelcache_base;
while (cur_pa < end_pa) {
vm_size_t range_size = end_pa - cur_pa;
vm_offset_t ptov_va = phystokv_range(cur_pa, &range_size);
if (ptov_va != cur_va) {
cur_pa += range_size;
cur_va += range_size;
continue;
}
unsigned int pai = (unsigned int)pa_index(cur_pa);
pv_entry_t **pv_h = pai_to_pvh(pai);
vm_offset_t pvh_flags = pvh_get_flags(pv_h);
if (__improbable(pvh_flags & PVH_FLAG_LOCKDOWN)) {
panic("pai %d already locked down", pai);
}
pvh_set_flags(pv_h, pvh_flags | PVH_FLAG_LOCKDOWN);
cur_pa += ARM_PGBYTES;
cur_va += ARM_PGBYTES;
}
#if defined(KERNEL_INTEGRITY_CTRR) && defined(CONFIG_XNUPOST)
extern uint64_t ctrr_ro_test;
extern uint64_t ctrr_nx_test;
pmap_paddr_t exclude_pages[] = {kvtophys((vm_offset_t)&ctrr_ro_test), kvtophys((vm_offset_t)&ctrr_nx_test)};
for (unsigned i = 0; i < (sizeof(exclude_pages) / sizeof(exclude_pages[0])); ++i) {
pv_entry_t **pv_h = pai_to_pvh(pa_index(exclude_pages[i]));
pvh_set_flags(pv_h, pvh_get_flags(pv_h) & ~PVH_FLAG_LOCKDOWN);
}
#endif
}
void
pmap_static_allocations_done(void)
{
pmap_paddr_t monitor_start_pa;
pmap_paddr_t monitor_end_pa;
monitor_start_pa = kvtophys((vm_offset_t)&bootstrap_pagetables);
monitor_end_pa = monitor_start_pa + BOOTSTRAP_TABLE_SIZE;
pa_set_range_xprr_perm(monitor_start_pa, monitor_end_pa, XPRR_KERN_RW_PERM, XPRR_KERN_RO_PERM);
monitor_start_pa = BootArgs->topOfKernelData;
monitor_end_pa = avail_start;
pa_set_range_xprr_perm(monitor_start_pa, monitor_end_pa, XPRR_KERN_RW_PERM, XPRR_PPL_RW_PERM);
monitor_start_pa = kvtophys((vm_offset_t)&ropagetable_begin);
monitor_end_pa = monitor_start_pa + ((vm_offset_t)&ropagetable_end - (vm_offset_t)&ropagetable_begin);
pa_set_range_monitor(monitor_start_pa, monitor_end_pa);
monitor_start_pa = kvtophys(segPPLDATAB);
monitor_end_pa = monitor_start_pa + segSizePPLDATA;
pa_set_range_xprr_perm(monitor_start_pa, monitor_end_pa, XPRR_KERN_RW_PERM, XPRR_PPL_RW_PERM);
monitor_start_pa = kvtophys(segPPLTEXTB);
monitor_end_pa = monitor_start_pa + segSizePPLTEXT;
pa_set_range_xprr_perm(monitor_start_pa, monitor_end_pa, XPRR_KERN_RX_PERM, XPRR_PPL_RX_PERM);
if (pmap_ppl_disable) {
vm_offset_t monitor_start_va = phystokv(ppl_cpu_save_area_start);
vm_offset_t monitor_end_va = monitor_start_va + (ppl_cpu_save_area_end - ppl_cpu_save_area_start);
pmap_set_range_xprr_perm(monitor_start_va, monitor_end_va, XPRR_PPL_RW_PERM, XPRR_KERN_RW_PERM);
}
if (segSizePPLDATACONST > 0) {
monitor_start_pa = kvtophys(segPPLDATACONSTB);
monitor_end_pa = monitor_start_pa + segSizePPLDATACONST;
pa_set_range_xprr_perm(monitor_start_pa, monitor_end_pa, XPRR_KERN_RO_PERM, XPRR_KERN_RO_PERM);
}
pa_set_range_xprr_perm(pmap_stacks_start_pa, pmap_stacks_end_pa, XPRR_PPL_RW_PERM, XPRR_KERN_RO_PERM);
pmap_lockdown_kc();
}
void
pmap_lockdown_ppl(void)
{
#error "XPRR configuration error"
}
#endif
void
pmap_virtual_space(
vm_offset_t *startp,
vm_offset_t *endp
)
{
*startp = virtual_space_start;
*endp = virtual_space_end;
}
boolean_t
pmap_virtual_region(
unsigned int region_select,
vm_map_offset_t *startp,
vm_map_size_t *size
)
{
boolean_t ret = FALSE;
#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
if (region_select == 0) {
#if (__ARM_VMSA__ > 7)
*startp = LOW_GLOBAL_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK;
#else
#error Unsupported configuration
#endif
#if defined(ARM_LARGE_MEMORY)
*size = ((KERNEL_PMAP_HEAP_RANGE_START - *startp) & ~PAGE_MASK);
#else
*size = ((VM_MAX_KERNEL_ADDRESS - *startp) & ~PAGE_MASK);
#endif
ret = TRUE;
}
#else
#if (__ARM_VMSA__ > 7)
unsigned long low_global_vr_mask = 0;
vm_map_size_t low_global_vr_size = 0;
#endif
if (region_select == 0) {
#if (__ARM_VMSA__ == 7)
*startp = gVirtBase & 0xFFC00000;
*size = ((virtual_space_start - (gVirtBase & 0xFFC00000)) + ~0xFFC00000) & 0xFFC00000;
#else
if (!TEST_PAGE_SIZE_4K) {
*startp = gVirtBase & 0xFFFFFFFFFE000000;
*size = ((virtual_space_start - (gVirtBase & 0xFFFFFFFFFE000000)) + ~0xFFFFFFFFFE000000) & 0xFFFFFFFFFE000000;
} else {
*startp = gVirtBase & 0xFFFFFFFFFF800000;
*size = ((virtual_space_start - (gVirtBase & 0xFFFFFFFFFF800000)) + ~0xFFFFFFFFFF800000) & 0xFFFFFFFFFF800000;
}
#endif
ret = TRUE;
}
if (region_select == 1) {
*startp = VREGION1_START;
*size = VREGION1_SIZE;
ret = TRUE;
}
#if (__ARM_VMSA__ > 7)
if (!TEST_PAGE_SIZE_4K) {
low_global_vr_mask = 0xFFFFFFFFFE000000;
low_global_vr_size = 0x2000000;
} else {
low_global_vr_mask = 0xFFFFFFFFFF800000;
low_global_vr_size = 0x800000;
}
if (((gVirtBase & low_global_vr_mask) != LOW_GLOBAL_BASE_ADDRESS) && (region_select == 2)) {
*startp = LOW_GLOBAL_BASE_ADDRESS;
*size = low_global_vr_size;
ret = TRUE;
}
if (region_select == 3) {
*startp = VM_MIN_KERNEL_ADDRESS;
*size = LOW_GLOBAL_BASE_ADDRESS - *startp;
ret = TRUE;
}
#endif
#endif
return ret;
}
static unsigned int avail_page_count = 0;
static bool need_ram_ranges_init = true;
#if defined(__arm64__)
pmap_paddr_t *bad_ram_pages = NULL;
unsigned int bad_ram_pages_count = 0;
static pmap_paddr_t *skip_pages;
static unsigned int skip_pages_count = 0;
#define MAX_BAD_RAM_PAGE_COUNT 64
static pmap_paddr_t bad_ram_pages_arr[MAX_BAD_RAM_PAGE_COUNT];
static void
parse_bad_ram_pages_boot_arg(void)
{
char buf[256] = {0};
char *s = buf;
char *end;
int count = 0;
pmap_paddr_t num;
extern uint64_t strtouq(const char *, char **, int);
if (!PE_parse_boot_arg_str("bad_ram_pages", buf, sizeof(buf))) {
goto done;
}
while (*s && count < MAX_BAD_RAM_PAGE_COUNT) {
num = (pmap_paddr_t)strtouq(s, &end, 0);
if (num == 0) {
break;
}
num &= ~PAGE_MASK;
bad_ram_pages_arr[count++] = gDramBase + num;
if (*end != ',') {
break;
}
s = end + 1;
}
done:
bad_ram_pages = bad_ram_pages_arr;
bad_ram_pages_count = count;
}
static int
pmap_paddr_cmp(void *a, void *b)
{
pmap_paddr_t *x = a;
pmap_paddr_t *y = b;
if (*x < *y) {
return -1;
}
return *x > *y;
}
#endif
bool
pmap_is_bad_ram(__unused ppnum_t ppn)
{
#if defined(__arm64__)
pmap_paddr_t pa = ptoa(ppn);
int low = 0;
int high = bad_ram_pages_count - 1;
int mid;
while (low <= high) {
mid = (low + high) / 2;
if (bad_ram_pages[mid] < pa) {
low = mid + 1;
} else if (bad_ram_pages[mid] > pa) {
high = mid - 1;
} else {
return true;
}
}
#endif
return false;
}
static void
initialize_ram_ranges(void)
{
pmap_paddr_t first = first_avail;
pmap_paddr_t end = avail_end;
assert(first <= end);
assert(first == (first & ~PAGE_MASK));
assert(end == (end & ~PAGE_MASK));
avail_page_count = atop(end - first);
#if defined(__arm64__)
parse_bad_ram_pages_boot_arg();
if (bad_ram_pages_count != 0) {
qsort(bad_ram_pages, bad_ram_pages_count, sizeof(*bad_ram_pages), (cmpfunc_t)pmap_paddr_cmp);
skip_pages = bad_ram_pages;
skip_pages_count = bad_ram_pages_count;
while (skip_pages_count > 0 && skip_pages[0] < first) {
--skip_pages_count;
++skip_pages;
}
while (skip_pages_count > 0 && skip_pages[skip_pages_count - 1] >= end) {
--skip_pages_count;
}
avail_page_count -= skip_pages_count;
}
#endif
need_ram_ranges_init = false;
}
unsigned int
pmap_free_pages(
void)
{
if (need_ram_ranges_init) {
initialize_ram_ranges();
}
return avail_page_count;
}
unsigned int
pmap_free_pages_span(
void)
{
if (need_ram_ranges_init) {
initialize_ram_ranges();
}
return (unsigned int)atop(avail_end - first_avail);
}
boolean_t
pmap_next_page_hi(
ppnum_t * pnum,
__unused boolean_t might_free)
{
return pmap_next_page(pnum);
}
boolean_t
pmap_next_page(
ppnum_t *pnum)
{
if (need_ram_ranges_init) {
initialize_ram_ranges();
}
#if defined(__arm64__)
while (skip_pages_count > 0 && first_avail == skip_pages[0]) {
first_avail += PAGE_SIZE;
++skip_pages;
--skip_pages_count;
}
#endif
if (first_avail != avail_end) {
*pnum = (ppnum_t)atop(first_avail);
first_avail += PAGE_SIZE;
assert(avail_page_count > 0);
--avail_page_count;
return TRUE;
}
assert(avail_page_count == 0);
return FALSE;
}
void
pmap_retire_page(
__unused ppnum_t pnum)
{
}
void
pmap_init(
void)
{
vm_protect(kernel_map, 0, PAGE_SIZE, TRUE, VM_PROT_NONE);
pmap_initialized = TRUE;
pmap_zone = zone_create_ext("pmap", sizeof(struct pmap),
ZC_ZFREE_CLEARMEM, ZONE_ID_PMAP, NULL);
_vm_object_allocate(mem_size, pmap_object);
pmap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
if ((uint32_t)maxproc > pmap_max_asids) {
maxproc = pmap_max_asids;
}
if ((uint32_t)hard_maxproc > pmap_max_asids) {
hard_maxproc = pmap_max_asids;
}
#if CONFIG_PGTRACE
pmap_pgtrace_init();
#endif
}
boolean_t
pmap_verify_free(
ppnum_t ppnum)
{
pv_entry_t **pv_h;
int pai;
pmap_paddr_t phys = ptoa(ppnum);
assert(phys != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return FALSE;
}
pai = (int)pa_index(phys);
pv_h = pai_to_pvh(pai);
return pvh_test_type(pv_h, PVH_TYPE_NULL);
}
#if MACH_ASSERT
void
pmap_assert_free(ppnum_t ppnum)
{
assertf(pmap_verify_free(ppnum), "page = 0x%x", ppnum);
(void)ppnum;
}
#endif
#if XNU_MONITOR
MARK_AS_PMAP_TEXT static void
pmap_ledger_alloc_init_internal(size_t size)
{
pmap_simple_lock(&pmap_ledger_lock);
if (pmap_ledger_alloc_initialized) {
panic("%s: already initialized, "
"size=%lu",
__func__,
size);
}
if ((size > sizeof(pmap_ledger_data_t)) ||
((sizeof(pmap_ledger_data_t) - size) % sizeof(struct ledger_entry))) {
panic("%s: size mismatch, expected %lu, "
"size=%lu",
__func__, PMAP_LEDGER_DATA_BYTES,
size);
}
pmap_ledger_alloc_initialized = true;
pmap_simple_unlock(&pmap_ledger_lock);
}
MARK_AS_PMAP_TEXT static ledger_t
pmap_ledger_alloc_internal(void)
{
pmap_paddr_t paddr;
uint64_t vaddr, vstart, vend;
uint64_t index;
ledger_t new_ledger;
uint64_t array_index;
pmap_simple_lock(&pmap_ledger_lock);
if (pmap_ledger_free_list == NULL) {
paddr = pmap_get_free_ppl_page();
if (paddr == 0) {
pmap_simple_unlock(&pmap_ledger_lock);
return NULL;
}
vstart = phystokv(paddr);
vend = vstart + PAGE_SIZE;
for (vaddr = vstart; (vaddr < vend) && ((vaddr + sizeof(pmap_ledger_t)) <= vend); vaddr += sizeof(pmap_ledger_t)) {
pmap_ledger_t *free_ledger;
index = pmap_ledger_ptr_array_free_index++;
if (index >= MAX_PMAP_LEDGERS) {
panic("%s: pmap_ledger_ptr_array is full, index=%llu",
__func__, index);
}
free_ledger = (pmap_ledger_t*)vaddr;
pmap_ledger_ptr_array[index] = free_ledger;
free_ledger->back_ptr = &pmap_ledger_ptr_array[index];
free_ledger->next = pmap_ledger_free_list;
pmap_ledger_free_list = free_ledger;
}
pa_set_range_xprr_perm(paddr, paddr + PAGE_SIZE, XPRR_PPL_RW_PERM, XPRR_KERN_RW_PERM);
}
new_ledger = (ledger_t)pmap_ledger_free_list;
pmap_ledger_free_list = pmap_ledger_free_list->next;
array_index = pmap_ledger_validate(new_ledger);
os_ref_init(&pmap_ledger_refcnt[array_index], NULL);
pmap_simple_unlock(&pmap_ledger_lock);
return new_ledger;
}
MARK_AS_PMAP_TEXT static void
pmap_ledger_free_internal(ledger_t ledger)
{
pmap_ledger_t* free_ledger;
free_ledger = (pmap_ledger_t*)ledger;
pmap_simple_lock(&pmap_ledger_lock);
uint64_t array_index = pmap_ledger_validate(ledger);
if (os_ref_release(&pmap_ledger_refcnt[array_index]) != 0) {
panic("%s: ledger still referenced, "
"ledger=%p",
__func__,
ledger);
}
free_ledger->next = pmap_ledger_free_list;
pmap_ledger_free_list = free_ledger;
pmap_simple_unlock(&pmap_ledger_lock);
}
static void
pmap_ledger_retain(ledger_t ledger)
{
pmap_simple_lock(&pmap_ledger_lock);
uint64_t array_index = pmap_ledger_validate(ledger);
os_ref_retain(&pmap_ledger_refcnt[array_index]);
pmap_simple_unlock(&pmap_ledger_lock);
}
static void
pmap_ledger_release(ledger_t ledger)
{
pmap_simple_lock(&pmap_ledger_lock);
uint64_t array_index = pmap_ledger_validate(ledger);
os_ref_release_live(&pmap_ledger_refcnt[array_index]);
pmap_simple_unlock(&pmap_ledger_lock);
}
void
pmap_ledger_alloc_init(size_t size)
{
pmap_ledger_alloc_init_ppl(size);
}
ledger_t
pmap_ledger_alloc(void)
{
ledger_t retval = NULL;
while ((retval = pmap_ledger_alloc_ppl()) == NULL) {
pmap_alloc_page_for_ppl(0);
}
return retval;
}
void
pmap_ledger_free(ledger_t ledger)
{
pmap_ledger_free_ppl(ledger);
}
#else
__dead2
void
pmap_ledger_alloc_init(size_t size)
{
panic("%s: unsupported, "
"size=%lu",
__func__, size);
}
__dead2
ledger_t
pmap_ledger_alloc(void)
{
panic("%s: unsupported",
__func__);
}
__dead2
void
pmap_ledger_free(ledger_t ledger)
{
panic("%s: unsupported, "
"ledger=%p",
__func__, ledger);
}
#endif
static vm_size_t
pmap_root_alloc_size(pmap_t pmap)
{
#if (__ARM_VMSA__ > 7)
#pragma unused(pmap)
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
unsigned int root_level = pt_attr_root_level(pt_attr);
return ((pt_attr_ln_index_mask(pt_attr, root_level) >> pt_attr_ln_shift(pt_attr, root_level)) + 1) * sizeof(tt_entry_t);
#else
(void)pmap;
return PMAP_ROOT_ALLOC_SIZE;
#endif
}
MARK_AS_PMAP_TEXT static pmap_t
pmap_create_options_internal(
ledger_t ledger,
vm_map_size_t size,
unsigned int flags,
kern_return_t *kr)
{
unsigned i;
unsigned tte_index_max;
pmap_t p;
bool is_64bit = flags & PMAP_CREATE_64BIT;
#if defined(HAS_APPLE_PAC)
bool disable_jop = flags & PMAP_CREATE_DISABLE_JOP;
#endif
kern_return_t local_kr = KERN_SUCCESS;
if (size != 0) {
return PMAP_NULL;
}
if (0 != (flags & ~PMAP_CREATE_KNOWN_FLAGS)) {
return PMAP_NULL;
}
#if XNU_MONITOR
if ((p = pmap_alloc_pmap()) == PMAP_NULL) {
local_kr = KERN_NO_SPACE;
goto pmap_create_fail;
}
if (ledger) {
pmap_ledger_validate(ledger);
pmap_ledger_retain(ledger);
}
#else
if ((p = (pmap_t) zalloc(pmap_zone)) == PMAP_NULL) {
local_kr = KERN_RESOURCE_SHORTAGE;
goto pmap_create_fail;
}
#endif
p->ledger = ledger;
p->pmap_vm_map_cs_enforced = false;
if (flags & PMAP_CREATE_64BIT) {
p->min = MACH_VM_MIN_ADDRESS;
p->max = MACH_VM_MAX_ADDRESS;
} else {
p->min = VM_MIN_ADDRESS;
p->max = VM_MAX_ADDRESS;
}
#if defined(HAS_APPLE_PAC)
p->disable_jop = disable_jop;
#endif
p->nested_region_true_start = 0;
p->nested_region_true_end = ~0;
os_atomic_init(&p->ref_count, 1);
p->gc_status = 0;
p->stamp = os_atomic_inc(&pmap_stamp, relaxed);
p->nx_enabled = TRUE;
p->is_64bit = is_64bit;
p->nested = FALSE;
p->nested_pmap = PMAP_NULL;
#if ARM_PARAMETERIZED_PMAP
p->pmap_pt_attr = native_pt_attr;
#endif
#if __ARM_MIXED_PAGE_SIZE__
if (flags & PMAP_CREATE_FORCE_4K_PAGES) {
p->pmap_pt_attr = &pmap_pt_attr_4k;
}
#endif
if (!pmap_get_pt_ops(p)->alloc_id(p)) {
local_kr = KERN_NO_SPACE;
goto id_alloc_fail;
}
pmap_lock_init(p);
memset((void *) &p->stats, 0, sizeof(p->stats));
p->tt_entry_free = (tt_entry_t *)0;
tte_index_max = ((unsigned)pmap_root_alloc_size(p) / sizeof(tt_entry_t));
#if (__ARM_VMSA__ == 7)
p->tte_index_max = tte_index_max;
#endif
#if XNU_MONITOR
p->tte = pmap_tt1_allocate(p, pmap_root_alloc_size(p), PMAP_TT_ALLOCATE_NOWAIT);
#else
p->tte = pmap_tt1_allocate(p, pmap_root_alloc_size(p), 0);
#endif
if (!(p->tte)) {
local_kr = KERN_RESOURCE_SHORTAGE;
goto tt1_alloc_fail;
}
p->ttep = ml_static_vtop((vm_offset_t)p->tte);
PMAP_TRACE(4, PMAP_CODE(PMAP__TTE), VM_KERNEL_ADDRHIDE(p), VM_KERNEL_ADDRHIDE(p->min), VM_KERNEL_ADDRHIDE(p->max), p->ttep);
for (i = 0; i < tte_index_max; i++) {
p->tte[i] = ARM_TTE_TYPE_FAULT;
}
FLUSH_PTE_RANGE(p->tte, p->tte + tte_index_max);
p->nested_region_addr = 0x0ULL;
p->nested_region_size = 0x0ULL;
p->nested_region_asid_bitmap = NULL;
p->nested_region_asid_bitmap_size = 0x0UL;
p->nested_has_no_bounds_ref = false;
p->nested_no_bounds_refcnt = 0;
p->nested_bounds_set = false;
#if MACH_ASSERT
p->pmap_stats_assert = TRUE;
p->pmap_pid = 0;
strlcpy(p->pmap_procname, "<nil>", sizeof(p->pmap_procname));
#endif
#if DEVELOPMENT || DEBUG
p->footprint_was_suspended = FALSE;
#endif
pmap_simple_lock(&pmaps_lock);
queue_enter(&map_pmap_list, p, pmap_t, pmaps);
pmap_simple_unlock(&pmaps_lock);
return p;
tt1_alloc_fail:
pmap_get_pt_ops(p)->free_id(p);
id_alloc_fail:
#if XNU_MONITOR
pmap_free_pmap(p);
if (ledger) {
pmap_ledger_release(ledger);
}
#else
zfree(pmap_zone, p);
#endif
pmap_create_fail:
#if XNU_MONITOR
pmap_pin_kernel_pages((vm_offset_t)kr, sizeof(*kr));
#endif
*kr = local_kr;
#if XNU_MONITOR
pmap_unpin_kernel_pages((vm_offset_t)kr, sizeof(*kr));
#endif
return PMAP_NULL;
}
pmap_t
pmap_create_options(
ledger_t ledger,
vm_map_size_t size,
unsigned int flags)
{
pmap_t pmap;
kern_return_t kr = KERN_SUCCESS;
PMAP_TRACE(1, PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, size, flags);
ledger_reference(ledger);
#if XNU_MONITOR
for (;;) {
pmap = pmap_create_options_ppl(ledger, size, flags, &kr);
if (kr != KERN_RESOURCE_SHORTAGE) {
break;
}
assert(pmap == PMAP_NULL);
pmap_alloc_page_for_ppl(0);
kr = KERN_SUCCESS;
}
#else
pmap = pmap_create_options_internal(ledger, size, flags, &kr);
#endif
if (pmap == PMAP_NULL) {
ledger_dereference(ledger);
}
PMAP_TRACE(1, PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END, VM_KERNEL_ADDRHIDE(pmap), PMAP_VASID(pmap), pmap->hw_asid);
return pmap;
}
#if XNU_MONITOR
#endif
#if MACH_ASSERT || XNU_MONITOR
MARK_AS_PMAP_TEXT static void
pmap_set_process_internal(
__unused pmap_t pmap,
__unused int pid,
__unused char *procname)
{
#if MACH_ASSERT
if (pmap == NULL) {
return;
}
VALIDATE_PMAP(pmap);
pmap->pmap_pid = pid;
strlcpy(pmap->pmap_procname, procname, sizeof(pmap->pmap_procname));
if (pmap_ledgers_panic_leeway) {
pmap->pmap_stats_assert = FALSE;
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.phys_footprint);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.internal);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.internal_compressed);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.iokit_mapped);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.alternate_accounting);
ledger_disable_panic_on_negative(pmap->ledger,
task_ledgers.alternate_accounting_compressed);
}
#endif
}
#endif
#if MACH_ASSERT
void
pmap_set_process(
pmap_t pmap,
int pid,
char *procname)
{
#if XNU_MONITOR
pmap_set_process_ppl(pmap, pid, procname);
#else
pmap_set_process_internal(pmap, pid, procname);
#endif
}
#endif
#if (__ARM_VMSA__ > 7)
MARK_AS_PMAP_TEXT static void
pmap_deallocate_all_leaf_tts(pmap_t pmap, tt_entry_t * first_ttep, unsigned level)
{
tt_entry_t tte = ARM_TTE_EMPTY;
tt_entry_t * ttep = NULL;
tt_entry_t * last_ttep = NULL;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
assert(level < pt_attr_leaf_level(pt_attr));
last_ttep = &first_ttep[ttn_index(pmap, pt_attr, ~0, level)];
for (ttep = first_ttep; ttep <= last_ttep; ttep++) {
tte = *ttep;
if (!(tte & ARM_TTE_VALID)) {
continue;
}
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
panic("%s: found block mapping, ttep=%p, tte=%p, "
"pmap=%p, first_ttep=%p, level=%u",
__FUNCTION__, ttep, (void *)tte,
pmap, first_ttep, level);
}
if (level < pt_attr_twig_level(pt_attr)) {
pmap_deallocate_all_leaf_tts(pmap, (tt_entry_t *)phystokv((tte) & ARM_TTE_TABLE_MASK), level + 1);
}
pmap_lock(pmap);
pmap_tte_deallocate(pmap, 0, 0, false, ttep, level);
pmap_unlock(pmap);
}
}
#endif
MARK_AS_PMAP_TEXT static void
pmap_destroy_internal(
pmap_t pmap)
{
if (pmap == PMAP_NULL) {
return;
}
VALIDATE_PMAP(pmap);
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
int32_t ref_count = os_atomic_dec(&pmap->ref_count, relaxed);
if (ref_count > 0) {
return;
} else if (ref_count < 0) {
panic("pmap %p: refcount underflow", pmap);
} else if (pmap == kernel_pmap) {
panic("pmap %p: attempt to destroy kernel pmap", pmap);
}
#if (__ARM_VMSA__ > 7)
pmap_unmap_sharedpage(pmap);
#endif
pmap_simple_lock(&pmaps_lock);
while (pmap->gc_status & PMAP_GC_INFLIGHT) {
pmap->gc_status |= PMAP_GC_WAIT;
assert_wait((event_t) &pmap->gc_status, THREAD_UNINT);
pmap_simple_unlock(&pmaps_lock);
(void) thread_block(THREAD_CONTINUE_NULL);
pmap_simple_lock(&pmaps_lock);
}
queue_remove(&map_pmap_list, pmap, pmap_t, pmaps);
pmap_simple_unlock(&pmaps_lock);
pmap_trim_self(pmap);
#if (__ARM_VMSA__ == 7)
unsigned int i = 0;
pt_entry_t *ttep;
pmap_lock(pmap);
for (i = 0; i < pmap->tte_index_max; i++) {
ttep = &pmap->tte[i];
if ((*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pmap_tte_deallocate(pmap, 0, 0, false, ttep, PMAP_TT_L1_LEVEL);
}
}
pmap_unlock(pmap);
#else
pmap_deallocate_all_leaf_tts(pmap, pmap->tte, pt_attr_root_level(pt_attr));
#endif
if (pmap->tte) {
#if (__ARM_VMSA__ == 7)
pmap_tt1_deallocate(pmap, pmap->tte, pmap->tte_index_max * sizeof(tt_entry_t), 0);
pmap->tte_index_max = 0;
#else
pmap_tt1_deallocate(pmap, pmap->tte, pmap_root_alloc_size(pmap), 0);
#endif
pmap->tte = (tt_entry_t *) NULL;
pmap->ttep = 0;
}
assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
if (__improbable(pmap->nested)) {
pmap_get_pt_ops(pmap)->flush_tlb_region_async(pmap->nested_region_addr, pmap->nested_region_size, pmap);
sync_tlb_flush();
} else {
pmap_get_pt_ops(pmap)->flush_tlb_async(pmap);
sync_tlb_flush();
pmap_get_pt_ops(pmap)->free_id(pmap);
pmap_destroy_internal(pmap->nested_pmap);
}
pmap_check_ledgers(pmap);
if (pmap->nested_region_asid_bitmap) {
#if XNU_MONITOR
pmap_pages_free(kvtophys((vm_offset_t)(pmap->nested_region_asid_bitmap)), PAGE_SIZE);
#else
kheap_free(KHEAP_DATA_BUFFERS, pmap->nested_region_asid_bitmap,
pmap->nested_region_asid_bitmap_size * sizeof(unsigned int));
#endif
}
#if XNU_MONITOR
if (pmap->ledger) {
pmap_ledger_release(pmap->ledger);
}
pmap_lock_destroy(pmap);
pmap_free_pmap(pmap);
#else
pmap_lock_destroy(pmap);
zfree(pmap_zone, pmap);
#endif
}
void
pmap_destroy(
pmap_t pmap)
{
ledger_t ledger;
PMAP_TRACE(1, PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(pmap), PMAP_VASID(pmap), pmap->hw_asid);
ledger = pmap->ledger;
#if XNU_MONITOR
pmap_destroy_ppl(pmap);
pmap_check_ledger_fields(ledger);
#else
pmap_destroy_internal(pmap);
#endif
ledger_dereference(ledger);
PMAP_TRACE(1, PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END);
}
MARK_AS_PMAP_TEXT static void
pmap_reference_internal(
pmap_t pmap)
{
if (pmap != PMAP_NULL) {
VALIDATE_PMAP(pmap);
os_atomic_inc(&pmap->ref_count, relaxed);
}
}
void
pmap_reference(
pmap_t pmap)
{
#if XNU_MONITOR
pmap_reference_ppl(pmap);
#else
pmap_reference_internal(pmap);
#endif
}
static tt_entry_t *
pmap_tt1_allocate(
pmap_t pmap,
vm_size_t size,
unsigned option)
{
tt_entry_t *tt1 = NULL;
tt_free_entry_t *tt1_free;
pmap_paddr_t pa;
vm_address_t va;
vm_address_t va_end;
kern_return_t ret;
if ((size < PAGE_SIZE) && (size != PMAP_ROOT_ALLOC_SIZE)) {
size = PAGE_SIZE;
}
pmap_simple_lock(&tt1_lock);
if ((size == PAGE_SIZE) && (free_page_size_tt_count != 0)) {
free_page_size_tt_count--;
tt1 = (tt_entry_t *)free_page_size_tt_list;
free_page_size_tt_list = ((tt_free_entry_t *)tt1)->next;
} else if ((size == 2 * PAGE_SIZE) && (free_two_page_size_tt_count != 0)) {
free_two_page_size_tt_count--;
tt1 = (tt_entry_t *)free_two_page_size_tt_list;
free_two_page_size_tt_list = ((tt_free_entry_t *)tt1)->next;
} else if ((size < PAGE_SIZE) && (free_tt_count != 0)) {
free_tt_count--;
tt1 = (tt_entry_t *)free_tt_list;
free_tt_list = (tt_free_entry_t *)((tt_free_entry_t *)tt1)->next;
}
pmap_simple_unlock(&tt1_lock);
if (tt1 != NULL) {
pmap_tt_ledger_credit(pmap, size);
return (tt_entry_t *)tt1;
}
ret = pmap_pages_alloc_zeroed(&pa, (unsigned)((size < PAGE_SIZE)? PAGE_SIZE : size), ((option & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0));
if (ret == KERN_RESOURCE_SHORTAGE) {
return (tt_entry_t *)0;
}
#if XNU_MONITOR
assert(pa);
#endif
if (size < PAGE_SIZE) {
va = phystokv(pa) + size;
tt_free_entry_t *local_free_list = (tt_free_entry_t*)va;
tt_free_entry_t *next_free = NULL;
for (va_end = phystokv(pa) + PAGE_SIZE; va < va_end; va = va + size) {
tt1_free = (tt_free_entry_t *)va;
tt1_free->next = next_free;
next_free = tt1_free;
}
pmap_simple_lock(&tt1_lock);
local_free_list->next = free_tt_list;
free_tt_list = next_free;
free_tt_count += ((PAGE_SIZE / size) - 1);
if (free_tt_count > free_tt_max) {
free_tt_max = free_tt_count;
}
pmap_simple_unlock(&tt1_lock);
}
OSAddAtomic((uint32_t)(size / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
OSAddAtomic64(size / PMAP_ROOT_ALLOC_SIZE, &alloc_tteroot_count);
pmap_tt_ledger_credit(pmap, size);
return (tt_entry_t *) phystokv(pa);
}
static void
pmap_tt1_deallocate(
pmap_t pmap,
tt_entry_t *tt,
vm_size_t size,
unsigned option)
{
tt_free_entry_t *tt_entry;
if ((size < PAGE_SIZE) && (size != PMAP_ROOT_ALLOC_SIZE)) {
size = PAGE_SIZE;
}
tt_entry = (tt_free_entry_t *)tt;
assert(not_in_kdp);
pmap_simple_lock(&tt1_lock);
if (size < PAGE_SIZE) {
free_tt_count++;
if (free_tt_count > free_tt_max) {
free_tt_max = free_tt_count;
}
tt_entry->next = free_tt_list;
free_tt_list = tt_entry;
}
if (size == PAGE_SIZE) {
free_page_size_tt_count++;
if (free_page_size_tt_count > free_page_size_tt_max) {
free_page_size_tt_max = free_page_size_tt_count;
}
tt_entry->next = free_page_size_tt_list;
free_page_size_tt_list = tt_entry;
}
if (size == 2 * PAGE_SIZE) {
free_two_page_size_tt_count++;
if (free_two_page_size_tt_count > free_two_page_size_tt_max) {
free_two_page_size_tt_max = free_two_page_size_tt_count;
}
tt_entry->next = free_two_page_size_tt_list;
free_two_page_size_tt_list = tt_entry;
}
if (option & PMAP_TT_DEALLOCATE_NOBLOCK) {
pmap_simple_unlock(&tt1_lock);
pmap_tt_ledger_debit(pmap, size);
return;
}
while (free_page_size_tt_count > FREE_PAGE_SIZE_TT_MAX) {
free_page_size_tt_count--;
tt = (tt_entry_t *)free_page_size_tt_list;
free_page_size_tt_list = ((tt_free_entry_t *)tt)->next;
pmap_simple_unlock(&tt1_lock);
pmap_pages_free(ml_static_vtop((vm_offset_t)tt), PAGE_SIZE);
OSAddAtomic(-(int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
pmap_simple_lock(&tt1_lock);
}
while (free_two_page_size_tt_count > FREE_TWO_PAGE_SIZE_TT_MAX) {
free_two_page_size_tt_count--;
tt = (tt_entry_t *)free_two_page_size_tt_list;
free_two_page_size_tt_list = ((tt_free_entry_t *)tt)->next;
pmap_simple_unlock(&tt1_lock);
pmap_pages_free(ml_static_vtop((vm_offset_t)tt), 2 * PAGE_SIZE);
OSAddAtomic(-2 * (int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count));
pmap_simple_lock(&tt1_lock);
}
pmap_simple_unlock(&tt1_lock);
pmap_tt_ledger_debit(pmap, size);
}
static kern_return_t
pmap_tt_allocate(
pmap_t pmap,
tt_entry_t **ttp,
unsigned int level,
unsigned int options)
{
pmap_paddr_t pa;
*ttp = NULL;
pmap_lock(pmap);
if ((tt_free_entry_t *)pmap->tt_entry_free != NULL) {
tt_free_entry_t *tt_free_cur, *tt_free_next;
tt_free_cur = ((tt_free_entry_t *)pmap->tt_entry_free);
tt_free_next = tt_free_cur->next;
tt_free_cur->next = NULL;
*ttp = (tt_entry_t *)tt_free_cur;
pmap->tt_entry_free = (tt_entry_t *)tt_free_next;
}
pmap_unlock(pmap);
if (*ttp == NULL) {
pt_desc_t *ptdp;
while (pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if (options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
while ((ptdp = ptd_alloc(pmap)) == NULL) {
if (options & PMAP_OPTIONS_NOWAIT) {
pmap_pages_free(pa, PAGE_SIZE);
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
if (level < pt_attr_leaf_level(pmap_get_pt_attr(pmap))) {
OSAddAtomic64(1, &alloc_ttepages_count);
OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count));
} else {
OSAddAtomic64(1, &alloc_ptepages_count);
OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count));
}
pmap_tt_ledger_credit(pmap, PAGE_SIZE);
PMAP_ZINFO_PALLOC(pmap, PAGE_SIZE);
pvh_update_head_unlocked(pai_to_pvh(pa_index(pa)), ptdp, PVH_TYPE_PTDP);
uint64_t pmap_page_size = pt_attr_page_size(pmap_get_pt_attr(pmap));
if (PAGE_SIZE > pmap_page_size) {
vm_address_t va;
vm_address_t va_end;
pmap_lock(pmap);
for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + pmap_page_size; va < va_end; va = va + pmap_page_size) {
((tt_free_entry_t *)va)->next = (tt_free_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = (tt_entry_t *)va;
}
pmap_unlock(pmap);
}
*ttp = (tt_entry_t *)phystokv(pa);
}
#if XNU_MONITOR
assert(*ttp);
#endif
return KERN_SUCCESS;
}
static void
pmap_tt_deallocate(
pmap_t pmap,
tt_entry_t *ttp,
unsigned int level)
{
pt_desc_t *ptdp;
ptd_info_t *ptd_info;
unsigned pt_acc_cnt;
unsigned i;
vm_offset_t free_page = 0;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
unsigned max_pt_index = PAGE_SIZE / pt_attr_page_size(pt_attr);
pmap_lock(pmap);
ptdp = ptep_get_ptd((vm_offset_t)ttp);
ptd_info = ptd_get_info(ptdp, ttp);
ptd_info->va = (vm_offset_t)-1;
if ((level < pt_attr_leaf_level(pt_attr)) && (ptd_info->refcnt == PT_DESC_REFCOUNT)) {
ptd_info->refcnt = 0;
}
if (ptd_info->refcnt != 0) {
panic("pmap_tt_deallocate(): ptdp %p, count %d\n", ptdp, ptd_info->refcnt);
}
ptd_info->refcnt = 0;
for (i = 0, pt_acc_cnt = 0; i < max_pt_index; i++) {
pt_acc_cnt += ptdp->ptd_info[i].refcnt;
}
if (pt_acc_cnt == 0) {
tt_free_entry_t *tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free;
unsigned pt_free_entry_cnt = 1;
while (pt_free_entry_cnt < max_pt_index && tt_free_list) {
tt_free_entry_t *tt_free_list_next;
tt_free_list_next = tt_free_list->next;
if ((((vm_offset_t)tt_free_list_next) - ((vm_offset_t)ttp & ~PAGE_MASK)) < PAGE_SIZE) {
pt_free_entry_cnt++;
}
tt_free_list = tt_free_list_next;
}
if (pt_free_entry_cnt == max_pt_index) {
tt_free_entry_t *tt_free_list_cur;
free_page = (vm_offset_t)ttp & ~PAGE_MASK;
tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free;
tt_free_list_cur = (tt_free_entry_t *)&pmap->tt_entry_free;
while (tt_free_list_cur) {
tt_free_entry_t *tt_free_list_next;
tt_free_list_next = tt_free_list_cur->next;
if ((((vm_offset_t)tt_free_list_next) - free_page) < PAGE_SIZE) {
tt_free_list->next = tt_free_list_next->next;
} else {
tt_free_list = tt_free_list_next;
}
tt_free_list_cur = tt_free_list_next;
}
} else {
((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = ttp;
}
} else {
((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free;
pmap->tt_entry_free = ttp;
}
pmap_unlock(pmap);
if (free_page != 0) {
ptd_deallocate(ptep_get_ptd((vm_offset_t)free_page));
*(pt_desc_t **)pai_to_pvh(pa_index(ml_static_vtop(free_page))) = NULL;
pmap_pages_free(ml_static_vtop(free_page), PAGE_SIZE);
if (level < pt_attr_leaf_level(pt_attr)) {
OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count));
} else {
OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count));
}
PMAP_ZINFO_PFREE(pmap, PAGE_SIZE);
pmap_tt_ledger_debit(pmap, PAGE_SIZE);
}
}
static void
pmap_tte_remove(
pmap_t pmap,
vm_offset_t va_start,
vm_offset_t va_end,
bool need_strong_sync,
tt_entry_t *ttep,
unsigned int level)
{
tt_entry_t tte = *ttep;
if (__improbable(tte == 0)) {
panic("%s: null tt_entry ttep==%p", __func__, ttep);
}
if (__improbable((level == pt_attr_twig_level(pmap_get_pt_attr(pmap))) &&
(ptep_get_info((pt_entry_t*)ttetokv(tte))->refcnt != 0))) {
panic("%s: non-zero pagetable refcount: pmap=%p ttep=%p ptd=%p refcnt=0x%x", __func__,
pmap, ttep, tte_get_ptd(tte), ptep_get_info((pt_entry_t*)ttetokv(tte))->refcnt);
}
#if (__ARM_VMSA__ == 7)
{
tt_entry_t *ttep_4M = (tt_entry_t *) ((vm_offset_t)ttep & 0xFFFFFFF0);
unsigned i;
for (i = 0; i < 4; i++, ttep_4M++) {
*ttep_4M = (tt_entry_t) 0;
}
FLUSH_PTE_RANGE_STRONG(ttep_4M - 4, ttep_4M);
}
#else
*ttep = (tt_entry_t) 0;
FLUSH_PTE_STRONG(ttep);
#endif
if (va_end > va_start) {
#if (__ARM_VMSA__ == 7)
flush_mmu_tlb_entry_async((va_start & ~ARM_TT_L1_PT_OFFMASK) | (pmap->hw_asid & 0xff));
flush_mmu_tlb_entry_async(((va_start & ~ARM_TT_L1_PT_OFFMASK) + ARM_TT_L1_SIZE) | (pmap->hw_asid & 0xff));
flush_mmu_tlb_entry_async(((va_start & ~ARM_TT_L1_PT_OFFMASK) + 2 * ARM_TT_L1_SIZE) | (pmap->hw_asid & 0xff));
flush_mmu_tlb_entry_async(((va_start & ~ARM_TT_L1_PT_OFFMASK) + 3 * ARM_TT_L1_SIZE) | (pmap->hw_asid & 0xff));
#endif
PMAP_UPDATE_TLBS(pmap, va_start, va_end, need_strong_sync);
}
}
static void
pmap_tte_deallocate(
pmap_t pmap,
vm_offset_t va_start,
vm_offset_t va_end,
bool need_strong_sync,
tt_entry_t *ttep,
unsigned int level)
{
pmap_paddr_t pa;
tt_entry_t tte;
pmap_assert_locked_w(pmap);
tte = *ttep;
#if MACH_ASSERT
if (tte_get_ptd(tte)->pmap != pmap) {
panic("%s: Passed in pmap doesn't own the page table to be deleted ptd=%p ptd->pmap=%p pmap=%p",
__func__, tte_get_ptd(tte), tte_get_ptd(tte)->pmap, pmap);
}
#endif
pmap_tte_remove(pmap, va_start, va_end, need_strong_sync, ttep, level);
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
uint64_t pmap_page_size = pt_attr_page_size(pmap_get_pt_attr(pmap));
#if MACH_ASSERT
pt_entry_t *pte_p = ((pt_entry_t *) (ttetokv(tte) & ~(pmap_page_size - 1)));
for (unsigned i = 0; i < (pmap_page_size / sizeof(*pte_p)); i++, pte_p++) {
if (__improbable(ARM_PTE_IS_COMPRESSED(*pte_p, pte_p))) {
panic_plain("%s: Found compressed mapping in soon to be deleted "
"L%d table tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx",
__func__, level + 1, (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p));
} else if (__improbable(((*pte_p) & ARM_PTE_TYPE_MASK) != ARM_PTE_TYPE_FAULT)) {
panic_plain("%s: Found valid mapping in soon to be deleted L%d "
"table tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx",
__func__, level + 1, (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p));
}
}
#endif
pmap_unlock(pmap);
pa = tte_to_pa(tte) & ~(pmap_page_size - 1);
pmap_tt_deallocate(pmap, (tt_entry_t *) phystokv(pa), level + 1);
pmap_lock(pmap);
}
}
static int
pmap_remove_range(
pmap_t pmap,
vm_map_address_t va,
pt_entry_t *bpte,
pt_entry_t *epte)
{
bool need_strong_sync = false;
int num_changed = pmap_remove_range_options(pmap, va, bpte, epte, NULL,
&need_strong_sync, PMAP_OPTIONS_REMOVE);
if (num_changed > 0) {
PMAP_UPDATE_TLBS(pmap, va,
va + (pt_attr_page_size(pmap_get_pt_attr(pmap)) * (epte - bpte)), need_strong_sync);
}
return num_changed;
}
#ifdef PVH_FLAG_EXEC
static void
pmap_set_ptov_ap(unsigned int pai __unused, unsigned int ap __unused, boolean_t flush_tlb_async __unused)
{
#if __ARM_PTE_PHYSMAP__
ASSERT_PVH_LOCKED(pai);
vm_offset_t kva = phystokv(vm_first_phys + (pmap_paddr_t)ptoa(pai));
pt_entry_t *pte_p = pmap_pte(kernel_pmap, kva);
pt_entry_t tmplate = *pte_p;
if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(ap)) {
return;
}
tmplate = (tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(ap);
#if (__ARM_VMSA__ > 7)
if (tmplate & ARM_PTE_HINT_MASK) {
panic("%s: physical aperture PTE %p has hint bit set, va=%p, pte=0x%llx",
__func__, pte_p, (void *)kva, tmplate);
}
#endif
WRITE_PTE_STRONG(pte_p, tmplate);
flush_mmu_tlb_region_asid_async(kva, PAGE_SIZE, kernel_pmap);
if (!flush_tlb_async) {
sync_tlb_flush();
}
#endif
}
#endif
static void
pmap_remove_pv(
pmap_t pmap,
pt_entry_t *cpte,
int pai,
int *num_internal,
int *num_alt_internal,
int *num_reusable,
int *num_external)
{
pv_entry_t **pv_h, **pve_pp;
pv_entry_t *pve_p;
ASSERT_NOT_HIBERNATING();
ASSERT_PVH_LOCKED(pai);
pv_h = pai_to_pvh(pai);
vm_offset_t pvh_flags = pvh_get_flags(pv_h);
#if XNU_MONITOR
if (__improbable(pvh_flags & PVH_FLAG_LOCKDOWN)) {
panic("%d is locked down (%#lx), cannot remove", pai, pvh_flags);
}
#endif
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
if (__improbable((cpte != pvh_ptep(pv_h)))) {
panic("%s: cpte=%p does not match pv_h=%p (%p), pai=0x%x\n", __func__, cpte, pv_h, pvh_ptep(pv_h), pai);
}
if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) {
assert(IS_INTERNAL_PAGE(pai));
(*num_internal)++;
(*num_alt_internal)++;
CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
} else if (IS_INTERNAL_PAGE(pai)) {
if (IS_REUSABLE_PAGE(pai)) {
(*num_reusable)++;
} else {
(*num_internal)++;
}
} else {
(*num_external)++;
}
pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_pp = pv_h;
pve_p = pvh_list(pv_h);
while (pve_p != PV_ENTRY_NULL &&
(pve_get_ptep(pve_p) != cpte)) {
pve_pp = pve_link_field(pve_p);
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
if (__improbable((pve_p == PV_ENTRY_NULL))) {
panic("%s: cpte=%p (pai=0x%x) not in pv_h=%p\n", __func__, cpte, pai, pv_h);
}
#if MACH_ASSERT
if ((pmap != NULL) && (kern_feature_override(KF_PMAPV_OVRD) == FALSE)) {
pv_entry_t *check_pve_p = PVE_NEXT_PTR(pve_next(pve_p));
while (check_pve_p != PV_ENTRY_NULL) {
if (pve_get_ptep(check_pve_p) == cpte) {
panic("%s: duplicate pve entry cpte=%p pmap=%p, pv_h=%p, pve_p=%p, pai=0x%x",
__func__, cpte, pmap, pv_h, pve_p, pai);
}
check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p));
}
}
#endif
if (IS_ALTACCT_PAGE(pai, pve_p)) {
assert(IS_INTERNAL_PAGE(pai));
(*num_internal)++;
(*num_alt_internal)++;
CLR_ALTACCT_PAGE(pai, pve_p);
} else if (IS_INTERNAL_PAGE(pai)) {
if (IS_REUSABLE_PAGE(pai)) {
(*num_reusable)++;
} else {
(*num_internal)++;
}
} else {
(*num_external)++;
}
pvh_remove(pv_h, pve_pp, pve_p);
pv_free_entry(pve_p);
if (!pvh_test_type(pv_h, PVH_TYPE_NULL)) {
pvh_set_flags(pv_h, pvh_flags);
}
} else {
panic("%s: unexpected PV head %p, cpte=%p pmap=%p pv_h=%p pai=0x%x",
__func__, *pv_h, cpte, pmap, pv_h, pai);
}
#ifdef PVH_FLAG_EXEC
if ((pvh_flags & PVH_FLAG_EXEC) && pvh_test_type(pv_h, PVH_TYPE_NULL)) {
pmap_set_ptov_ap(pai, AP_RWNA, FALSE);
}
#endif
}
static int
pmap_remove_range_options(
pmap_t pmap,
vm_map_address_t va,
pt_entry_t *bpte,
pt_entry_t *epte,
vm_map_address_t *eva,
bool *need_strong_sync __unused,
int options)
{
pt_entry_t *cpte;
size_t npages = 0;
int num_removed, num_unwired;
int num_pte_changed;
int pai = 0;
pmap_paddr_t pa;
int num_external, num_internal, num_reusable;
int num_alt_internal;
uint64_t num_compressed, num_alt_compressed;
int16_t refcnt = 0;
pmap_assert_locked_w(pmap);
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
uint64_t pmap_page_size = PAGE_RATIO * pt_attr_page_size(pt_attr);
if (__improbable((uintptr_t)epte > (((uintptr_t)bpte + pmap_page_size) & ~(pmap_page_size - 1)))) {
panic("%s: PTE range [%p, %p) in pmap %p crosses page table boundary", __func__, bpte, epte, pmap);
}
num_removed = 0;
num_unwired = 0;
num_pte_changed = 0;
num_external = 0;
num_internal = 0;
num_reusable = 0;
num_compressed = 0;
num_alt_internal = 0;
num_alt_compressed = 0;
for (cpte = bpte; cpte < epte;
cpte += PAGE_RATIO, va += pmap_page_size) {
pt_entry_t spte;
boolean_t managed = FALSE;
if (__improbable((eva != NULL) && npages++ && pmap_pending_preemption())) {
*eva = va;
break;
}
spte = *((volatile pt_entry_t*)cpte);
#if CONFIG_PGTRACE
if (pgtrace_enabled) {
pmap_pgtrace_remove_clone(pmap, pte_to_pa(spte), va);
}
#endif
while (!managed) {
if (pmap != kernel_pmap &&
(options & PMAP_OPTIONS_REMOVE) &&
(ARM_PTE_IS_COMPRESSED(spte, cpte))) {
num_compressed++;
if (spte & ARM_PTE_COMPRESSED_ALT) {
num_alt_compressed++;
}
WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT);
--refcnt;
spte = *((volatile pt_entry_t*)cpte);
}
pa = pte_to_pa(spte);
if (!pa_valid(pa)) {
#if XNU_MONITOR
unsigned int cacheattr = pmap_cache_attributes((ppnum_t)atop(pa));
#endif
#if XNU_MONITOR
if (__improbable((cacheattr & PP_ATTR_MONITOR) &&
(pte_to_xprr_perm(spte) != XPRR_KERN_RO_PERM) && !pmap_ppl_disable)) {
panic("%s: attempt to remove mapping of writable PPL-protected I/O address 0x%llx",
__func__, (uint64_t)pa);
}
#endif
break;
}
pai = (int)pa_index(pa);
LOCK_PVH(pai);
spte = *((volatile pt_entry_t*)cpte);
pa = pte_to_pa(spte);
if (pai == (int)pa_index(pa)) {
managed = TRUE;
break; }
UNLOCK_PVH(pai);
}
if (ARM_PTE_IS_COMPRESSED(*cpte, cpte)) {
continue;
}
if (*cpte != ARM_PTE_TYPE_FAULT) {
assertf(!ARM_PTE_IS_COMPRESSED(*cpte, cpte), "unexpected compressed pte %p (=0x%llx)", cpte, (uint64_t)*cpte);
assertf((*cpte & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE, "invalid pte %p (=0x%llx)", cpte, (uint64_t)*cpte);
#if MACH_ASSERT
if (managed && (pmap != kernel_pmap) && (ptep_get_va(cpte) != va)) {
panic("pmap_remove_range_options(): VA mismatch: cpte=%p ptd=%p pte=0x%llx va=0x%llx, cpte va=0x%llx",
cpte, ptep_get_ptd(cpte), (uint64_t)*cpte, (uint64_t)va, (uint64_t)ptep_get_va(cpte));
}
#endif
WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT);
num_pte_changed++;
}
if ((spte != ARM_PTE_TYPE_FAULT) &&
(pmap != kernel_pmap)) {
assertf(!ARM_PTE_IS_COMPRESSED(spte, cpte), "unexpected compressed pte %p (=0x%llx)", cpte, (uint64_t)spte);
assertf((spte & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE, "invalid pte %p (=0x%llx)", cpte, (uint64_t)spte);
--refcnt;
}
if (pte_is_wired(spte)) {
pte_set_wired(pmap, cpte, 0);
num_unwired++;
}
if (!managed) {
continue;
}
pmap_remove_pv(pmap, cpte, pai, &num_internal, &num_alt_internal, &num_reusable, &num_external);
UNLOCK_PVH(pai);
num_removed++;
}
OSAddAtomic(-num_removed, (SInt32 *) &pmap->stats.resident_count);
pmap_ledger_debit(pmap, task_ledgers.phys_mem, num_removed * pmap_page_size);
if (pmap != kernel_pmap) {
if ((refcnt != 0) && (OSAddAtomic16(refcnt, (SInt16 *) &(ptep_get_info(bpte)->refcnt)) <= 0)) {
panic("pmap_remove_range_options: over-release of ptdp %p for pte [%p, %p)", ptep_get_ptd(bpte), bpte, epte);
}
OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count);
if (num_external) {
__assert_only int32_t orig_external = OSAddAtomic(-num_external, &pmap->stats.external);
PMAP_STATS_ASSERTF(orig_external >= num_external,
pmap,
"pmap=%p bpte=%p epte=%p num_external=%d stats.external=%d",
pmap, bpte, epte, num_external, orig_external);
}
if (num_internal) {
__assert_only int32_t orig_internal = OSAddAtomic(-num_internal, &pmap->stats.internal);
PMAP_STATS_ASSERTF(orig_internal >= num_internal,
pmap,
"pmap=%p bpte=%p epte=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
pmap, bpte, epte,
num_internal, orig_internal,
num_reusable, pmap->stats.reusable);
}
if (num_reusable) {
__assert_only int32_t orig_reusable = OSAddAtomic(-num_reusable, &pmap->stats.reusable);
PMAP_STATS_ASSERTF(orig_reusable >= num_reusable,
pmap,
"pmap=%p bpte=%p epte=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
pmap, bpte, epte,
num_internal, pmap->stats.internal,
num_reusable, orig_reusable);
}
if (num_compressed) {
__assert_only uint64_t orig_compressed = OSAddAtomic64(-num_compressed, &pmap->stats.compressed);
PMAP_STATS_ASSERTF(orig_compressed >= num_compressed,
pmap,
"pmap=%p bpte=%p epte=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld",
pmap, bpte, epte, num_compressed, num_alt_compressed,
orig_compressed);
}
pmap_ledger_debit(pmap, task_ledgers.wired_mem, (num_unwired) * pmap_page_size);
pmap_ledger_debit(pmap, task_ledgers.internal, (num_internal) * pmap_page_size);
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, (num_alt_internal) * pmap_page_size);
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting_compressed, (num_alt_compressed) * pmap_page_size);
pmap_ledger_debit(pmap, task_ledgers.internal_compressed, (num_compressed) * pmap_page_size);
pmap_ledger_debit(pmap, task_ledgers.phys_footprint,
((num_internal -
num_alt_internal) +
(num_compressed -
num_alt_compressed)) * pmap_page_size);
}
if (num_pte_changed > 0) {
FLUSH_PTE_RANGE_STRONG(bpte, epte);
}
return num_pte_changed;
}
void
pmap_remove(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end)
{
pmap_remove_options(pmap, start, end, PMAP_OPTIONS_REMOVE);
}
MARK_AS_PMAP_TEXT static vm_map_address_t
pmap_remove_options_internal(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
int options)
{
vm_map_address_t eva = end;
pt_entry_t *bpte, *epte;
pt_entry_t *pte_p;
tt_entry_t *tte_p;
int remove_count = 0;
bool need_strong_sync = false;
if (__improbable(end < start)) {
panic("%s: invalid address range %p, %p", __func__, (void*)start, (void*)end);
}
VALIDATE_PMAP(pmap);
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
pmap_lock(pmap);
tte_p = pmap_tte(pmap, start);
if (tte_p == (tt_entry_t *) NULL) {
goto done;
}
if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[pte_index(pmap, pt_attr, start)];
epte = bpte + ((end - start) >> pt_attr_leaf_shift(pt_attr));
remove_count = pmap_remove_range_options(pmap, start, bpte, epte, &eva,
&need_strong_sync, options);
if ((pmap != kernel_pmap) && (pmap->nested == FALSE) && (ptep_get_info(pte_p)->refcnt == 0)) {
pmap_tte_deallocate(pmap, start, eva, need_strong_sync, tte_p, pt_attr_twig_level(pt_attr));
remove_count = 0; }
}
done:
pmap_unlock(pmap);
if (remove_count > 0) {
PMAP_UPDATE_TLBS(pmap, start, eva, need_strong_sync);
}
return eva;
}
void
pmap_remove_options(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
int options)
{
vm_map_address_t va;
if (pmap == PMAP_NULL) {
return;
}
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
PMAP_TRACE(2, PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start),
VM_KERNEL_ADDRHIDE(end));
#if MACH_ASSERT
if ((start | end) & pt_attr_leaf_offmask(pt_attr)) {
panic("pmap_remove_options() pmap %p start 0x%llx end 0x%llx\n",
pmap, (uint64_t)start, (uint64_t)end);
}
if ((end < start) || (start < pmap->min) || (end > pmap->max)) {
panic("pmap_remove_options(): invalid address range, pmap=%p, start=0x%llx, end=0x%llx\n",
pmap, (uint64_t)start, (uint64_t)end);
}
#endif
assert(get_preemption_level() == 0);
va = start;
while (va < end) {
vm_map_address_t l;
l = ((va + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr));
if (l > end) {
l = end;
}
#if XNU_MONITOR
va = pmap_remove_options_ppl(pmap, va, l, options);
pmap_ledger_check_balance(pmap);
#else
va = pmap_remove_options_internal(pmap, va, l, options);
#endif
}
PMAP_TRACE(2, PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END);
}
void
pmap_remove_some_phys(
__unused pmap_t map,
__unused ppnum_t pn)
{
}
void
pmap_switch_user(thread_t thread, vm_map_t new_map)
{
pmap_t new_pmap = new_map->pmap;
thread->map = new_map;
pmap_set_pmap(new_pmap, thread);
}
void
pmap_set_pmap(
pmap_t pmap,
#if !__ARM_USER_PROTECT__
__unused
#endif
thread_t thread)
{
pmap_switch(pmap);
#if __ARM_USER_PROTECT__
thread->machine.uptw_ttb = ((unsigned int) pmap->ttep) | TTBR_SETUP;
thread->machine.asid = pmap->hw_asid;
#endif
}
static void
pmap_flush_core_tlb_asid_async(pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
flush_core_tlb_asid_async(pmap->hw_asid);
#else
flush_core_tlb_asid_async(((uint64_t) pmap->hw_asid) << TLBI_ASID_SHIFT);
#endif
}
static inline bool
pmap_user_ttb_is_clear(void)
{
#if (__ARM_VMSA__ > 7)
return get_mmu_ttb() == (invalid_ttep & TTBR_BADDR_MASK);
#else
return get_mmu_ttb() == kernel_pmap->ttep;
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_switch_internal(
pmap_t pmap)
{
VALIDATE_PMAP(pmap);
pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data();
uint16_t asid_index = pmap->hw_asid;
bool do_asid_flush = false;
if (__improbable((asid_index == 0) && (pmap != kernel_pmap))) {
panic("%s: attempt to activate pmap with invalid ASID %p", __func__, pmap);
}
#if __ARM_KERNEL_PROTECT__
asid_index >>= 1;
#endif
#if (__ARM_VMSA__ > 7)
pmap_t last_nested_pmap = cpu_data_ptr->cpu_nested_pmap;
__unused const pt_attr_t *last_nested_pmap_attr = cpu_data_ptr->cpu_nested_pmap_attr;
__unused vm_map_address_t last_nested_region_addr = cpu_data_ptr->cpu_nested_region_addr;
__unused vm_map_offset_t last_nested_region_size = cpu_data_ptr->cpu_nested_region_size;
bool do_shared_region_flush = ((pmap != kernel_pmap) && (last_nested_pmap != NULL) && (pmap->nested_pmap != last_nested_pmap));
bool break_before_make = do_shared_region_flush;
#else
bool do_shared_region_flush = false;
bool break_before_make = false;
#endif
if ((pmap_max_asids > MAX_HW_ASIDS) && (asid_index > 0)) {
asid_index -= 1;
pmap_update_plru(asid_index);
assert(asid_index < (sizeof(cpu_data_ptr->cpu_sw_asids) / sizeof(*cpu_data_ptr->cpu_sw_asids)));
uint8_t new_sw_asid = pmap->sw_asid;
uint8_t last_sw_asid = cpu_data_ptr->cpu_sw_asids[asid_index];
if (new_sw_asid != last_sw_asid) {
cpu_data_ptr->cpu_sw_asids[asid_index] = new_sw_asid;
do_asid_flush = true;
break_before_make = true;
}
}
#if __ARM_MIXED_PAGE_SIZE__
if (pmap_get_pt_attr(pmap)->pta_tcr_value != get_tcr()) {
break_before_make = true;
}
#endif
if (__improbable(break_before_make && !pmap_user_ttb_is_clear())) {
PMAP_TRACE(1, PMAP_CODE(PMAP__CLEAR_USER_TTB), VM_KERNEL_ADDRHIDE(pmap), PMAP_VASID(pmap), pmap->hw_asid);
pmap_clear_user_ttb_internal();
}
#if (__ARM_VMSA__ > 7)
if (__improbable(do_shared_region_flush)) {
#if __ARM_RANGE_TLBI__
uint64_t page_shift_prev = pt_attr_leaf_shift(last_nested_pmap_attr);
vm_map_offset_t npages_prev = last_nested_region_size >> page_shift_prev;
if (npages_prev <= ARM64_TLB_RANGE_PAGES) {
flush_core_tlb_allrange_async(generate_rtlbi_param((ppnum_t)npages_prev, 0, last_nested_region_addr, page_shift_prev));
} else {
do_asid_flush = false;
flush_core_tlb_async();
}
#else
do_asid_flush = false;
flush_core_tlb_async();
#endif // __ARM_RANGE_TLBI__
}
#endif // (__ARM_VMSA__ > 7)
if (__improbable(do_asid_flush)) {
pmap_flush_core_tlb_asid_async(pmap);
#if DEVELOPMENT || DEBUG
os_atomic_inc(&pmap_asid_flushes, relaxed);
#endif
}
if (__improbable(do_asid_flush || do_shared_region_flush)) {
sync_tlb_flush();
}
pmap_switch_user_ttb_internal(pmap);
}
void
pmap_switch(
pmap_t pmap)
{
PMAP_TRACE(1, PMAP_CODE(PMAP__SWITCH) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(pmap), PMAP_VASID(pmap), pmap->hw_asid);
#if XNU_MONITOR
pmap_switch_ppl(pmap);
#else
pmap_switch_internal(pmap);
#endif
PMAP_TRACE(1, PMAP_CODE(PMAP__SWITCH) | DBG_FUNC_END);
}
void
pmap_require(pmap_t pmap)
{
#if XNU_MONITOR
VALIDATE_PMAP(pmap);
#else
if (pmap != kernel_pmap) {
zone_id_require(ZONE_ID_PMAP, sizeof(struct pmap), pmap);
}
#endif
}
void
pmap_page_protect(
ppnum_t ppnum,
vm_prot_t prot)
{
pmap_page_protect_options(ppnum, prot, 0, NULL);
}
MARK_AS_PMAP_TEXT static void
pmap_page_protect_options_with_flush_range(
ppnum_t ppnum,
vm_prot_t prot,
unsigned int options,
pmap_tlb_flush_range_t *flush_range)
{
pmap_paddr_t phys = ptoa(ppnum);
pv_entry_t **pv_h;
pv_entry_t **pve_pp;
pv_entry_t *pve_p;
pv_entry_t *pveh_p;
pv_entry_t *pvet_p;
pt_entry_t *pte_p;
pv_entry_t *new_pve_p;
pt_entry_t *new_pte_p;
vm_offset_t pvh_flags;
int pai;
boolean_t remove;
boolean_t set_NX;
boolean_t tlb_flush_needed = FALSE;
unsigned int pvh_cnt = 0;
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return;
}
switch (prot) {
case VM_PROT_ALL:
return;
case VM_PROT_READ:
case VM_PROT_READ | VM_PROT_EXECUTE:
remove = FALSE;
break;
default:
options = options & ~PMAP_OPTIONS_NOFLUSH;
remove = TRUE;
break;
}
pai = (int)pa_index(phys);
LOCK_PVH(pai);
pv_h = pai_to_pvh(pai);
pvh_flags = pvh_get_flags(pv_h);
#if XNU_MONITOR
if (__improbable(remove && (pvh_flags & PVH_FLAG_LOCKDOWN))) {
panic("%d is locked down (%#llx), cannot remove", pai, pvh_get_flags(pv_h));
}
#endif
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
pve_pp = pv_h;
pveh_p = PV_ENTRY_NULL;
pvet_p = PV_ENTRY_NULL;
new_pve_p = PV_ENTRY_NULL;
new_pte_p = PT_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
pveh_p = pve_p;
}
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va = 0;
pmap_t pmap = NULL;
pt_entry_t tmplate = ARM_PTE_TYPE_FAULT;
boolean_t update = FALSE;
if (pve_p != PV_ENTRY_NULL) {
pte_p = pve_get_ptep(pve_p);
}
#ifdef PVH_FLAG_IOMMU
if ((vm_offset_t)pte_p & PVH_FLAG_IOMMU) {
#if XNU_MONITOR
if (__improbable(pvh_flags & PVH_FLAG_LOCKDOWN)) {
panic("pmap_page_protect: ppnum 0x%x locked down, cannot be owned by iommu 0x%llx, pve_p=%p",
ppnum, (uint64_t)pte_p & ~PVH_FLAG_IOMMU, pve_p);
}
#endif
if (remove) {
if (options & PMAP_OPTIONS_COMPRESSOR) {
panic("pmap_page_protect: attempt to compress ppnum 0x%x owned by iommu 0x%llx, pve_p=%p",
ppnum, (uint64_t)pte_p & ~PVH_FLAG_IOMMU, pve_p);
}
if (pve_p != PV_ENTRY_NULL) {
pv_entry_t *temp_pve_p = PVE_NEXT_PTR(pve_next(pve_p));
pvh_remove(pv_h, pve_pp, pve_p);
pveh_p = pvh_list(pv_h);
pve_next(pve_p) = new_pve_p;
new_pve_p = pve_p;
pve_p = temp_pve_p;
continue;
} else {
new_pte_p = pte_p;
break;
}
}
goto protect_skip_pve;
}
#endif
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
if (pte_p == PT_ENTRY_NULL) {
panic("pmap_page_protect: pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, va=0x%llx ppnum: 0x%x\n",
pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)va, ppnum);
} else if ((pmap == NULL) || (atop(pte_to_pa(*pte_p)) != ppnum)) {
#if MACH_ASSERT
if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) {
pv_entry_t *check_pve_p = pveh_p;
while (check_pve_p != PV_ENTRY_NULL) {
if ((check_pve_p != pve_p) && (pve_get_ptep(check_pve_p) == pte_p)) {
panic("pmap_page_protect: duplicate pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n",
pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum);
}
check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p));
}
}
#endif
panic("pmap_page_protect: bad pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n",
pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum);
}
#if DEVELOPMENT || DEBUG
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
#else
if ((prot & VM_PROT_EXECUTE))
#endif
{
set_NX = FALSE;
} else {
set_NX = TRUE;
}
if (remove) {
boolean_t is_altacct = FALSE;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
pt_entry_t spte = *pte_p;
if (IS_ALTACCT_PAGE(pai, pve_p)) {
is_altacct = TRUE;
} else {
is_altacct = FALSE;
}
if (pte_is_wired(spte)) {
pte_set_wired(pmap, pte_p, 0);
spte = *pte_p;
if (pmap != kernel_pmap) {
pmap_ledger_debit(pmap, task_ledgers.wired_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
}
}
if (spte != ARM_PTE_TYPE_FAULT &&
pmap != kernel_pmap &&
(options & PMAP_OPTIONS_COMPRESSOR) &&
IS_INTERNAL_PAGE(pai)) {
assert(!ARM_PTE_IS_COMPRESSED(*pte_p, pte_p));
tmplate = ARM_PTE_COMPRESSED;
if (is_altacct) {
tmplate |= ARM_PTE_COMPRESSED_ALT;
is_altacct = TRUE;
}
} else {
tmplate = ARM_PTE_TYPE_FAULT;
}
if (spte != tmplate) {
WRITE_PTE_STRONG(pte_p, tmplate);
update = TRUE;
}
if ((spte != ARM_PTE_TYPE_FAULT) &&
(tmplate == ARM_PTE_TYPE_FAULT) &&
(pmap != kernel_pmap)) {
if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_info(pte_p)->refcnt)) <= 0) {
panic("pmap_page_protect_options(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
}
}
pvh_cnt++;
pmap_ledger_debit(pmap, task_ledgers.phys_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count);
#if MACH_ASSERT
if (options & PMAP_OPTIONS_COMPRESSOR) {
assert(IS_INTERNAL_PAGE(pai));
}
#endif
if (pmap != kernel_pmap) {
if (IS_REUSABLE_PAGE(pai) &&
IS_INTERNAL_PAGE(pai) &&
!is_altacct) {
__assert_only int32_t orig_reusable = OSAddAtomic(-1, &pmap->stats.reusable);
PMAP_STATS_ASSERTF(orig_reusable > 0, pmap, "stats.reusable %d", orig_reusable);
} else if (IS_INTERNAL_PAGE(pai)) {
__assert_only int32_t orig_internal = OSAddAtomic(-1, &pmap->stats.internal);
PMAP_STATS_ASSERTF(orig_internal > 0, pmap, "stats.internal %d", orig_internal);
} else {
__assert_only int32_t orig_external = OSAddAtomic(-1, &pmap->stats.external);
PMAP_STATS_ASSERTF(orig_external > 0, pmap, "stats.external %d", orig_external);
}
if ((options & PMAP_OPTIONS_COMPRESSOR) &&
IS_INTERNAL_PAGE(pai)) {
OSAddAtomic64(+1, &pmap->stats.compressed);
PMAP_STATS_PEAK(pmap->stats.compressed);
pmap->stats.compressed_lifetime++;
}
if (IS_ALTACCT_PAGE(pai, pve_p)) {
assert(IS_INTERNAL_PAGE(pai));
pmap_ledger_debit(pmap, task_ledgers.internal, pt_attr_page_size(pt_attr) * PAGE_RATIO);
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, pt_attr_page_size(pt_attr) * PAGE_RATIO);
if (options & PMAP_OPTIONS_COMPRESSOR) {
pmap_ledger_credit(pmap, task_ledgers.internal_compressed, pt_attr_page_size(pt_attr) * PAGE_RATIO);
pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
CLR_ALTACCT_PAGE(pai, pve_p);
} else if (IS_REUSABLE_PAGE(pai)) {
assert(IS_INTERNAL_PAGE(pai));
if (options & PMAP_OPTIONS_COMPRESSOR) {
pmap_ledger_credit(pmap, task_ledgers.internal_compressed, pt_attr_page_size(pt_attr) * PAGE_RATIO);
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
} else if (IS_INTERNAL_PAGE(pai)) {
pmap_ledger_debit(pmap, task_ledgers.internal, pt_attr_page_size(pt_attr) * PAGE_RATIO);
if (options & PMAP_OPTIONS_COMPRESSOR) {
pmap_ledger_credit(pmap, task_ledgers.internal_compressed, pt_attr_page_size(pt_attr) * PAGE_RATIO);
} else {
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
} else {
}
}
if (pve_p != PV_ENTRY_NULL) {
assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p)));
}
} else {
pt_entry_t spte;
const pt_attr_t *const pt_attr = pmap_get_pt_attr(pmap);
spte = *pte_p;
if (pmap == kernel_pmap) {
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
} else {
tmplate = ((spte & ~ARM_PTE_APMASK) | pt_attr_leaf_ro(pt_attr));
}
pte_set_was_writeable(tmplate, false);
if (set_NX) {
tmplate |= pt_attr_leaf_xn(pt_attr);
}
if (*pte_p != ARM_PTE_TYPE_FAULT &&
!ARM_PTE_IS_COMPRESSED(*pte_p, pte_p) &&
*pte_p != tmplate) {
if (options & PMAP_OPTIONS_NOFLUSH) {
WRITE_PTE_FAST(pte_p, tmplate);
} else {
WRITE_PTE_STRONG(pte_p, tmplate);
}
update = TRUE;
}
}
if (update && !(options & PMAP_OPTIONS_NOFLUSH)) {
if (remove || !flush_range ||
((flush_range->ptfr_pmap != pmap) || va >= flush_range->ptfr_end || va < flush_range->ptfr_start)) {
pmap_get_pt_ops(pmap)->flush_tlb_region_async(va,
pt_attr_page_size(pmap_get_pt_attr(pmap)) * PAGE_RATIO, pmap);
}
tlb_flush_needed = TRUE;
}
#ifdef PVH_FLAG_IOMMU
protect_skip_pve:
#endif
pte_p = PT_ENTRY_NULL;
pvet_p = pve_p;
if (pve_p != PV_ENTRY_NULL) {
if (remove) {
assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p)));
}
pve_pp = pve_link_field(pve_p);
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
#ifdef PVH_FLAG_EXEC
if (remove && (pvh_get_flags(pv_h) & PVH_FLAG_EXEC)) {
pmap_set_ptov_ap(pai, AP_RWNA, tlb_flush_needed);
}
#endif
if (remove) {
if (new_pve_p != PV_ENTRY_NULL) {
pvh_update_head(pv_h, new_pve_p, PVH_TYPE_PVEP);
pvh_set_flags(pv_h, pvh_flags);
} else if (new_pte_p != PT_ENTRY_NULL) {
pvh_update_head(pv_h, new_pte_p, PVH_TYPE_PTEP);
pvh_set_flags(pv_h, pvh_flags);
} else {
pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL);
}
}
if (flush_range && tlb_flush_needed) {
if (!remove) {
flush_range->ptfr_flush_needed = true;
tlb_flush_needed = FALSE;
}
}
if (remove && tlb_flush_needed) {
sync_tlb_flush();
}
UNLOCK_PVH(pai);
if (!remove && tlb_flush_needed) {
sync_tlb_flush();
}
if (remove && (pvet_p != PV_ENTRY_NULL)) {
pv_list_free(pveh_p, pvet_p, pvh_cnt, pv_kern_low_water_mark);
}
}
MARK_AS_PMAP_TEXT static void
pmap_page_protect_options_internal(
ppnum_t ppnum,
vm_prot_t prot,
unsigned int options,
void *arg)
{
if (arg != NULL) {
options &= ~PMAP_OPTIONS_NOFLUSH;
}
pmap_page_protect_options_with_flush_range(ppnum, prot, options, NULL);
}
void
pmap_page_protect_options(
ppnum_t ppnum,
vm_prot_t prot,
unsigned int options,
void *arg)
{
pmap_paddr_t phys = ptoa(ppnum);
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return;
}
if (prot == VM_PROT_ALL) {
return;
}
PMAP_TRACE(2, PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, ppnum, prot);
#if XNU_MONITOR
pmap_page_protect_options_ppl(ppnum, prot, options, arg);
#else
pmap_page_protect_options_internal(ppnum, prot, options, arg);
#endif
PMAP_TRACE(2, PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END);
}
#if __has_feature(ptrauth_calls) && defined(XNU_TARGET_OS_OSX)
MARK_AS_PMAP_TEXT void
pmap_disable_user_jop_internal(pmap_t pmap)
{
if (pmap == kernel_pmap) {
panic("%s: called with kernel_pmap\n", __func__);
}
VALIDATE_PMAP(pmap);
pmap->disable_jop = true;
}
void
pmap_disable_user_jop(pmap_t pmap)
{
#if XNU_MONITOR
pmap_disable_user_jop_ppl(pmap);
#else
pmap_disable_user_jop_internal(pmap);
#endif
}
#endif
bool
pmap_has_prot_policy(__unused pmap_t pmap, __unused bool translated_allow_execute, __unused vm_prot_t prot)
{
return false;
}
void
pmap_protect(
pmap_t pmap,
vm_map_address_t b,
vm_map_address_t e,
vm_prot_t prot)
{
pmap_protect_options(pmap, b, e, prot, 0, NULL);
}
MARK_AS_PMAP_TEXT static vm_map_address_t
pmap_protect_options_internal(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
vm_prot_t prot,
unsigned int options,
__unused void *args)
{
const pt_attr_t *const pt_attr = pmap_get_pt_attr(pmap);
tt_entry_t *tte_p;
pt_entry_t *bpte_p, *epte_p;
pt_entry_t *pte_p;
boolean_t set_NX = TRUE;
#if (__ARM_VMSA__ > 7)
boolean_t set_XO = FALSE;
#endif
boolean_t should_have_removed = FALSE;
bool need_strong_sync = false;
if (__improbable((end < start) || (end > ((start + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr))))) {
panic("%s: invalid address range %p, %p", __func__, (void*)start, (void*)end);
}
#if DEVELOPMENT || DEBUG
if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
if ((prot & VM_PROT_ALL) == VM_PROT_NONE) {
should_have_removed = TRUE;
}
} else
#endif
{
switch (prot) {
#if (__ARM_VMSA__ > 7)
case VM_PROT_EXECUTE:
set_XO = TRUE;
OS_FALLTHROUGH;
#endif
case VM_PROT_READ:
case VM_PROT_READ | VM_PROT_EXECUTE:
break;
case VM_PROT_READ | VM_PROT_WRITE:
case VM_PROT_ALL:
return end;
default:
should_have_removed = TRUE;
}
}
if (should_have_removed) {
panic("%s: should have been a remove operation, "
"pmap=%p, start=%p, end=%p, prot=%#x, options=%#x, args=%p",
__FUNCTION__,
pmap, (void *)start, (void *)end, prot, options, args);
}
#if DEVELOPMENT || DEBUG
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
#else
if ((prot & VM_PROT_EXECUTE))
#endif
{
set_NX = FALSE;
} else {
set_NX = TRUE;
}
const uint64_t pmap_page_size = PAGE_RATIO * pt_attr_page_size(pt_attr);
vm_map_address_t va = start;
unsigned int npages = 0;
VALIDATE_PMAP(pmap);
pmap_lock(pmap);
tte_p = pmap_tte(pmap, start);
if ((tte_p != (tt_entry_t *) NULL) && (*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
bpte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte_p = &bpte_p[pte_index(pmap, pt_attr, start)];
epte_p = bpte_p + ((end - start) >> pt_attr_leaf_shift(pt_attr));
pte_p = bpte_p;
for (pte_p = bpte_p;
pte_p < epte_p;
pte_p += PAGE_RATIO, va += pmap_page_size) {
++npages;
if (__improbable(!(npages % PMAP_DEFAULT_PREEMPTION_CHECK_PAGE_INTERVAL) &&
pmap_pending_preemption())) {
break;
}
pt_entry_t spte;
#if DEVELOPMENT || DEBUG
boolean_t force_write = FALSE;
#endif
spte = *((volatile pt_entry_t*)pte_p);
if ((spte == ARM_PTE_TYPE_FAULT) ||
ARM_PTE_IS_COMPRESSED(spte, pte_p)) {
continue;
}
pmap_paddr_t pa;
int pai = 0;
boolean_t managed = FALSE;
while (!managed) {
pa = pte_to_pa(spte);
if (!pa_valid(pa)) {
break;
}
pai = (int)pa_index(pa);
LOCK_PVH(pai);
spte = *((volatile pt_entry_t*)pte_p);
pa = pte_to_pa(spte);
if (pai == (int)pa_index(pa)) {
managed = TRUE;
break; }
UNLOCK_PVH(pai);
}
if ((spte == ARM_PTE_TYPE_FAULT) ||
ARM_PTE_IS_COMPRESSED(spte, pte_p)) {
continue;
}
pt_entry_t tmplate;
if (pmap == kernel_pmap) {
#if DEVELOPMENT || DEBUG
if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) {
force_write = TRUE;
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA));
} else
#endif
{
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
}
} else {
#if DEVELOPMENT || DEBUG
if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) {
force_write = TRUE;
tmplate = ((spte & ~ARM_PTE_APMASK) | pt_attr_leaf_rw(pt_attr));
} else
#endif
{
tmplate = ((spte & ~ARM_PTE_APMASK) | pt_attr_leaf_ro(pt_attr));
}
}
if (set_NX) {
tmplate |= pt_attr_leaf_xn(pt_attr);
} else {
#if (__ARM_VMSA__ > 7)
if (pmap == kernel_pmap) {
tmplate |= ARM_PTE_NX;
} else {
tmplate |= pt_attr_leaf_x(pt_attr);
if (set_XO) {
tmplate &= ~ARM_PTE_APMASK;
tmplate |= pt_attr_leaf_rona(pt_attr);
}
}
#endif
}
#if DEVELOPMENT || DEBUG
if (force_write) {
if (managed) {
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
tmplate |= ARM_PTE_AF;
if (IS_REFFAULT_PAGE(pai)) {
CLR_REFFAULT_PAGE(pai);
}
if (IS_MODFAULT_PAGE(pai)) {
CLR_MODFAULT_PAGE(pai);
}
}
} else if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
if (managed) {
pa_set_bits(pa, PP_ATTR_REFERENCED);
tmplate |= ARM_PTE_AF;
if (IS_REFFAULT_PAGE(pai)) {
CLR_REFFAULT_PAGE(pai);
}
}
}
#endif
pte_set_was_writeable(tmplate, false);
WRITE_PTE_FAST(pte_p, tmplate);
if (managed) {
ASSERT_PVH_LOCKED(pai);
UNLOCK_PVH(pai);
}
}
FLUSH_PTE_RANGE_STRONG(bpte_p, pte_p);
PMAP_UPDATE_TLBS(pmap, start, va, need_strong_sync);
} else {
va = end;
}
pmap_unlock(pmap);
return va;
}
void
pmap_protect_options(
pmap_t pmap,
vm_map_address_t b,
vm_map_address_t e,
vm_prot_t prot,
unsigned int options,
__unused void *args)
{
vm_map_address_t l, beg;
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if ((b | e) & pt_attr_leaf_offmask(pt_attr)) {
panic("pmap_protect_options() pmap %p start 0x%llx end 0x%llx\n",
pmap, (uint64_t)b, (uint64_t)e);
}
assert(get_preemption_level() == 0);
#if DEVELOPMENT || DEBUG
if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) {
if ((prot & VM_PROT_ALL) == VM_PROT_NONE) {
pmap_remove_options(pmap, b, e, options);
return;
}
} else
#endif
{
switch (prot) {
case VM_PROT_EXECUTE:
case VM_PROT_READ:
case VM_PROT_READ | VM_PROT_EXECUTE:
break;
case VM_PROT_READ | VM_PROT_WRITE:
case VM_PROT_ALL:
return;
default:
pmap_remove_options(pmap, b, e, options);
return;
}
}
PMAP_TRACE(2, PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(b),
VM_KERNEL_ADDRHIDE(e));
beg = b;
while (beg < e) {
l = ((beg + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr));
if (l > e) {
l = e;
}
#if XNU_MONITOR
beg = pmap_protect_options_ppl(pmap, beg, l, prot, options, args);
#else
beg = pmap_protect_options_internal(pmap, beg, l, prot, options, args);
#endif
}
PMAP_TRACE(2, PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END);
}
kern_return_t
pmap_map_block(
pmap_t pmap,
addr64_t va,
ppnum_t pa,
uint32_t size,
vm_prot_t prot,
int attr,
__unused unsigned int flags)
{
kern_return_t kr;
addr64_t original_va = va;
uint32_t page;
for (page = 0; page < size; page++) {
kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
if (kr != KERN_SUCCESS) {
panic("%s: failed pmap_enter, "
"pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x",
__FUNCTION__,
pmap, va, pa, size, prot, flags);
pmap_remove(pmap, original_va, va - original_va);
return kr;
}
va += PAGE_SIZE;
pa++;
}
return KERN_SUCCESS;
}
kern_return_t
pmap_map_block_addr(
pmap_t pmap,
addr64_t va,
pmap_paddr_t pa,
uint32_t size,
vm_prot_t prot,
int attr,
__unused unsigned int flags)
{
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
const uint64_t pmap_page_size = pt_attr_page_size(pt_attr) * PAGE_RATIO;
kern_return_t kr;
uint32_t page;
for (page = 0; page < size; page++) {
kr = pmap_enter_addr(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE);
if (kr != KERN_SUCCESS) {
panic("%s: failed pmap_enter_addr, "
"pmap=%p, va=%#llx, pa=%llu, size=%u, prot=%#x, flags=%#x",
__FUNCTION__,
pmap, va, (uint64_t)pa, size, prot, flags);
}
va += pmap_page_size;
pa += pmap_page_size;
}
return KERN_SUCCESS;
}
kern_return_t
pmap_enter_addr(
pmap_t pmap,
vm_map_address_t v,
pmap_paddr_t pa,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired)
{
return pmap_enter_options_addr(pmap, v, pa, prot, fault_type, flags, wired, 0, NULL);
}
kern_return_t
pmap_enter(
pmap_t pmap,
vm_map_address_t v,
ppnum_t pn,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired)
{
return pmap_enter_addr(pmap, v, ((pmap_paddr_t)pn) << PAGE_SHIFT, prot, fault_type, flags, wired);
}
static inline void
pmap_enter_pte(pmap_t pmap, pt_entry_t *pte_p, pt_entry_t pte, vm_map_address_t v)
{
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if (pmap != kernel_pmap && ((pte & ARM_PTE_WIRED) != (*pte_p & ARM_PTE_WIRED))) {
SInt16 *ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_info(pte_p)->wiredcnt);
if (pte & ARM_PTE_WIRED) {
OSAddAtomic16(1, ptd_wiredcnt_ptr);
pmap_ledger_credit(pmap, task_ledgers.wired_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
OSAddAtomic(1, (SInt32 *) &pmap->stats.wired_count);
} else {
OSAddAtomic16(-1, ptd_wiredcnt_ptr);
pmap_ledger_debit(pmap, task_ledgers.wired_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
}
}
if (*pte_p != ARM_PTE_TYPE_FAULT &&
!ARM_PTE_IS_COMPRESSED(*pte_p, pte_p)) {
WRITE_PTE_STRONG(pte_p, pte);
PMAP_UPDATE_TLBS(pmap, v, v + (pt_attr_page_size(pt_attr) * PAGE_RATIO), false);
} else {
WRITE_PTE(pte_p, pte);
__builtin_arm_isb(ISB_SY);
}
PMAP_TRACE(4 + pt_attr_leaf_level(pt_attr), PMAP_CODE(PMAP__TTE), VM_KERNEL_ADDRHIDE(pmap),
VM_KERNEL_ADDRHIDE(v), VM_KERNEL_ADDRHIDE(v + (pt_attr_page_size(pt_attr) * PAGE_RATIO)), pte);
}
MARK_AS_PMAP_TEXT static pt_entry_t
wimg_to_pte(unsigned int wimg)
{
pt_entry_t pte;
switch (wimg & (VM_WIMG_MASK)) {
case VM_WIMG_IO:
case VM_WIMG_RT:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_POSTED:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_POSTED_REORDERED:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED_REORDERED);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_POSTED_COMBINED_REORDERED:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED_COMBINED_REORDERED);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_WCOMB:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB);
pte |= ARM_PTE_NX | ARM_PTE_PNX;
break;
case VM_WIMG_WTHRU:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITETHRU);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
break;
case VM_WIMG_COPYBACK:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
break;
case VM_WIMG_INNERWBACK:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_INNERWRITEBACK);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_INNER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
break;
default:
pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
}
return pte;
}
static pv_alloc_return_t
pmap_enter_pv(
pmap_t pmap,
pt_entry_t *pte_p,
int pai,
unsigned int options,
pv_entry_t **pve_p,
boolean_t *is_altacct)
{
pv_entry_t **pv_h;
pv_h = pai_to_pvh(pai);
boolean_t first_cpu_mapping;
ASSERT_NOT_HIBERNATING();
ASSERT_PVH_LOCKED(pai);
vm_offset_t pvh_flags = pvh_get_flags(pv_h);
#if XNU_MONITOR
if (__improbable(pvh_flags & PVH_FLAG_LOCKDOWN)) {
panic("%d is locked down (%#lx), cannot enter", pai, pvh_flags);
}
#endif
#ifdef PVH_FLAG_CPU
first_cpu_mapping = ((pmap != NULL) && !(pvh_flags & PVH_FLAG_CPU));
if (first_cpu_mapping) {
pvh_flags |= PVH_FLAG_CPU;
}
#else
first_cpu_mapping = pvh_test_type(pv_h, PVH_TYPE_NULL);
#endif
if (first_cpu_mapping) {
if (options & PMAP_OPTIONS_INTERNAL) {
SET_INTERNAL_PAGE(pai);
} else {
CLR_INTERNAL_PAGE(pai);
}
if ((options & PMAP_OPTIONS_INTERNAL) &&
(options & PMAP_OPTIONS_REUSABLE)) {
SET_REUSABLE_PAGE(pai);
} else {
CLR_REUSABLE_PAGE(pai);
}
}
if (pvh_test_type(pv_h, PVH_TYPE_NULL)) {
pvh_update_head(pv_h, pte_p, PVH_TYPE_PTEP);
if (pmap != NULL && pmap != kernel_pmap &&
((options & PMAP_OPTIONS_ALT_ACCT) ||
PMAP_FOOTPRINT_SUSPENDED(pmap)) &&
IS_INTERNAL_PAGE(pai)) {
SET_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
*is_altacct = TRUE;
} else {
CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
}
} else {
pv_alloc_return_t ret;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pt_entry_t *pte1_p;
pte1_p = pvh_ptep(pv_h);
pvh_set_flags(pv_h, pvh_flags);
if ((*pve_p == PV_ENTRY_NULL) && ((ret = pv_alloc(pmap, pai, pve_p)) != PV_ALLOC_SUCCESS)) {
return ret;
}
pve_set_ptep(*pve_p, pte1_p);
(*pve_p)->pve_next = PV_ENTRY_NULL;
if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) {
CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL);
SET_ALTACCT_PAGE(pai, *pve_p);
}
pvh_update_head(pv_h, *pve_p, PVH_TYPE_PVEP);
*pve_p = PV_ENTRY_NULL;
} else if (!pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
panic("%s: unexpected PV head %p, pte_p=%p pmap=%p pv_h=%p",
__func__, *pv_h, pte_p, pmap, pv_h);
}
pvh_set_flags(pv_h, pvh_flags);
if ((*pve_p == PV_ENTRY_NULL) && ((ret = pv_alloc(pmap, pai, pve_p)) != PV_ALLOC_SUCCESS)) {
return ret;
}
pve_set_ptep(*pve_p, pte_p);
(*pve_p)->pve_next = PV_ENTRY_NULL;
pvh_add(pv_h, *pve_p);
if (pmap != NULL && pmap != kernel_pmap &&
((options & PMAP_OPTIONS_ALT_ACCT) ||
PMAP_FOOTPRINT_SUSPENDED(pmap)) &&
IS_INTERNAL_PAGE(pai)) {
SET_ALTACCT_PAGE(pai, *pve_p);
*is_altacct = TRUE;
}
*pve_p = PV_ENTRY_NULL;
}
pvh_set_flags(pv_h, pvh_flags);
return PV_ALLOC_SUCCESS;
}
MARK_AS_PMAP_TEXT static kern_return_t
pmap_enter_options_internal(
pmap_t pmap,
vm_map_address_t v,
pmap_paddr_t pa,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options)
{
ppnum_t pn = (ppnum_t)atop(pa);
pt_entry_t pte;
pt_entry_t spte;
pt_entry_t *pte_p;
pv_entry_t *pve_p;
boolean_t set_NX;
boolean_t set_XO = FALSE;
boolean_t refcnt_updated;
boolean_t wiredcnt_updated;
unsigned int wimg_bits;
boolean_t was_compressed, was_alt_compressed;
kern_return_t kr = KERN_SUCCESS;
VALIDATE_PMAP(pmap);
#if XNU_MONITOR
if (__improbable((options & PMAP_OPTIONS_NOWAIT) == 0)) {
panic("pmap_enter_options() called without PMAP_OPTIONS_NOWAIT set");
}
#endif
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if ((v) & pt_attr_leaf_offmask(pt_attr)) {
panic("pmap_enter_options() pmap %p v 0x%llx\n",
pmap, (uint64_t)v);
}
if ((pa) & pt_attr_leaf_offmask(pt_attr)) {
panic("pmap_enter_options() pmap %p pa 0x%llx\n",
pmap, (uint64_t)pa);
}
pa &= ARM_PTE_PAGE_MASK;
if ((prot & VM_PROT_EXECUTE) && (pmap == kernel_pmap)) {
#if defined(KERNEL_INTEGRITY_CTRR) && defined(CONFIG_XNUPOST)
extern vm_offset_t ctrr_test_page;
if (__probable(v != ctrr_test_page))
#endif
panic("pmap_enter_options(): attempt to add executable mapping to kernel_pmap");
}
#if DEVELOPMENT || DEBUG
if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled)
#else
if ((prot & VM_PROT_EXECUTE))
#endif
{
set_NX = FALSE;
} else {
set_NX = TRUE;
}
#if (__ARM_VMSA__ > 7)
if (prot == VM_PROT_EXECUTE) {
set_XO = TRUE;
}
#endif
assert(pn != vm_page_fictitious_addr);
refcnt_updated = FALSE;
wiredcnt_updated = FALSE;
pve_p = PV_ENTRY_NULL;
was_compressed = FALSE;
was_alt_compressed = FALSE;
pmap_lock(pmap);
while ((pte_p = pmap_pte(pmap, v)) == PT_ENTRY_NULL) {
pmap_unlock(pmap);
kr = pmap_expand(pmap, v, options, pt_attr_leaf_level(pt_attr));
if (kr != KERN_SUCCESS) {
return kr;
}
pmap_lock(pmap);
}
if (options & PMAP_OPTIONS_NOENTER) {
pmap_unlock(pmap);
return KERN_SUCCESS;
}
Pmap_enter_retry:
spte = *pte_p;
if (ARM_PTE_IS_COMPRESSED(spte, pte_p) && !refcnt_updated) {
assert(pmap != kernel_pmap);
OSAddAtomic64(-1, &pmap->stats.compressed);
pmap_ledger_debit(pmap, task_ledgers.internal_compressed,
pt_attr_page_size(pt_attr) * PAGE_RATIO);
was_compressed = TRUE;
if (spte & ARM_PTE_COMPRESSED_ALT) {
was_alt_compressed = TRUE;
pmap_ledger_debit(pmap, task_ledgers.alternate_accounting_compressed, pt_attr_page_size(pt_attr) * PAGE_RATIO);
} else {
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
WRITE_PTE_FAST(pte_p, ARM_PTE_TYPE_FAULT);
spte = ARM_PTE_TYPE_FAULT;
refcnt_updated = TRUE;
}
if ((spte != ARM_PTE_TYPE_FAULT) && (pte_to_pa(spte) != pa)) {
pmap_remove_range(pmap, v, pte_p, pte_p + PAGE_RATIO);
}
pte = pa_to_pte(pa) | ARM_PTE_TYPE;
if (wired) {
pte |= ARM_PTE_WIRED;
}
if (set_NX) {
pte |= pt_attr_leaf_xn(pt_attr);
} else {
#if (__ARM_VMSA__ > 7)
if (pmap == kernel_pmap) {
pte |= ARM_PTE_NX;
} else {
pte |= pt_attr_leaf_x(pt_attr);
}
#endif
}
if (pmap == kernel_pmap) {
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
if (prot & VM_PROT_WRITE) {
pte |= ARM_PTE_AP(AP_RWNA);
pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED);
} else {
pte |= ARM_PTE_AP(AP_RONA);
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
#if (__ARM_VMSA__ == 7)
if ((_COMM_PAGE_BASE_ADDRESS <= v) && (v < _COMM_PAGE_BASE_ADDRESS + _COMM_PAGE_AREA_LENGTH)) {
pte = (pte & ~(ARM_PTE_APMASK)) | ARM_PTE_AP(AP_RORO);
}
#endif
} else {
if (!pmap->nested) {
pte |= ARM_PTE_NG;
} else if ((pmap->nested_region_asid_bitmap)
&& (v >= pmap->nested_region_addr)
&& (v < (pmap->nested_region_addr + pmap->nested_region_size))) {
unsigned int index = (unsigned int)((v - pmap->nested_region_addr) >> pt_attr_twig_shift(pt_attr));
if ((pmap->nested_region_asid_bitmap)
&& testbit(index, (int *)pmap->nested_region_asid_bitmap)) {
pte |= ARM_PTE_NG;
}
}
#if MACH_ASSERT
if (pmap->nested_pmap != NULL) {
vm_map_address_t nest_vaddr;
pt_entry_t *nest_pte_p;
nest_vaddr = v;
if ((nest_vaddr >= pmap->nested_region_addr)
&& (nest_vaddr < (pmap->nested_region_addr + pmap->nested_region_size))
&& ((nest_pte_p = pmap_pte(pmap->nested_pmap, nest_vaddr)) != PT_ENTRY_NULL)
&& (*nest_pte_p != ARM_PTE_TYPE_FAULT)
&& (!ARM_PTE_IS_COMPRESSED(*nest_pte_p, nest_pte_p))
&& (((*nest_pte_p) & ARM_PTE_NG) != ARM_PTE_NG)) {
unsigned int index = (unsigned int)((v - pmap->nested_region_addr) >> pt_attr_twig_shift(pt_attr));
if ((pmap->nested_pmap->nested_region_asid_bitmap)
&& !testbit(index, (int *)pmap->nested_pmap->nested_region_asid_bitmap)) {
panic("pmap_enter(): Global attribute conflict nest_pte_p=%p pmap=%p v=0x%llx spte=0x%llx \n",
nest_pte_p, pmap, (uint64_t)v, (uint64_t)*nest_pte_p);
}
}
}
#endif
if (prot & VM_PROT_WRITE) {
if (pa_valid(pa) && (!pa_test_bits(pa, PP_ATTR_MODIFIED))) {
assert(!pmap->nested);
if (fault_type & VM_PROT_WRITE) {
if (set_XO) {
pte |= pt_attr_leaf_rwna(pt_attr);
} else {
pte |= pt_attr_leaf_rw(pt_attr);
}
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
} else {
if (set_XO) {
pte |= pt_attr_leaf_rona(pt_attr);
} else {
pte |= pt_attr_leaf_ro(pt_attr);
}
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODFAULT);
pte_set_was_writeable(pte, true);
}
} else {
if (set_XO) {
pte |= pt_attr_leaf_rwna(pt_attr);
} else {
pte |= pt_attr_leaf_rw(pt_attr);
}
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
} else {
if (set_XO) {
pte |= pt_attr_leaf_rona(pt_attr);
} else {
pte |= pt_attr_leaf_ro(pt_attr);;
}
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
}
pte |= ARM_PTE_AF;
volatile uint16_t *refcnt = NULL;
volatile uint16_t *wiredcnt = NULL;
if (pmap != kernel_pmap) {
ptd_info_t *ptd_info = ptep_get_info(pte_p);
refcnt = &ptd_info->refcnt;
wiredcnt = &ptd_info->wiredcnt;
if (!wiredcnt_updated) {
OSAddAtomic16(1, (volatile int16_t*)wiredcnt);
wiredcnt_updated = TRUE;
}
if (!refcnt_updated) {
OSAddAtomic16(1, (volatile int16_t*)refcnt);
refcnt_updated = TRUE;
}
}
if (pa_valid(pa)) {
int pai;
boolean_t is_altacct, is_internal;
is_internal = FALSE;
is_altacct = FALSE;
pai = (int)pa_index(pa);
LOCK_PVH(pai);
Pmap_enter_loop:
if ((flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT))) {
wimg_bits = (flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT));
} else {
wimg_bits = pmap_cache_attributes(pn);
}
pte &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);
pte |= pmap_get_pt_ops(pmap)->wimg_to_pte(wimg_bits);
#if XNU_MONITOR
if (__improbable(pa_test_monitor(pa))) {
panic("%s: page belongs to PPL, "
"pmap=%p, v=0x%llx, pa=%p, prot=0x%x, fault_type=0x%x, flags=0x%x, wired=%u, options=0x%x",
__FUNCTION__,
pmap, v, (void*)pa, prot, fault_type, flags, wired, options);
}
if (__improbable(pvh_get_flags(pai_to_pvh(pai)) & PVH_FLAG_LOCKDOWN)) {
panic("%s: page locked down, "
"pmap=%p, v=0x%llx, pa=%p, prot=0x%x, fault_type=0x%x, flags=0x%x, wired=%u, options=0x%x",
__FUNCTION__,
pmap, v, (void *)pa, prot, fault_type, flags, wired, options);
}
#endif
if (pte == *pte_p) {
UNLOCK_PVH(pai);
goto Pmap_enter_cleanup;
} else if (pte_to_pa(*pte_p) == pa) {
pmap_enter_pte(pmap, pte_p, pte, v);
UNLOCK_PVH(pai);
goto Pmap_enter_cleanup;
} else if (*pte_p != ARM_PTE_TYPE_FAULT) {
UNLOCK_PVH(pai);
goto Pmap_enter_retry;
}
pv_alloc_return_t pv_status = pmap_enter_pv(pmap, pte_p, pai, options, &pve_p, &is_altacct);
if (pv_status == PV_ALLOC_RETRY) {
goto Pmap_enter_loop;
} else if (pv_status == PV_ALLOC_FAIL) {
UNLOCK_PVH(pai);
kr = KERN_RESOURCE_SHORTAGE;
goto Pmap_enter_cleanup;
}
pmap_enter_pte(pmap, pte_p, pte, v);
if (pmap != kernel_pmap) {
if (IS_REUSABLE_PAGE(pai) &&
!is_altacct) {
assert(IS_INTERNAL_PAGE(pai));
OSAddAtomic(+1, &pmap->stats.reusable);
PMAP_STATS_PEAK(pmap->stats.reusable);
} else if (IS_INTERNAL_PAGE(pai)) {
OSAddAtomic(+1, &pmap->stats.internal);
PMAP_STATS_PEAK(pmap->stats.internal);
is_internal = TRUE;
} else {
OSAddAtomic(+1, &pmap->stats.external);
PMAP_STATS_PEAK(pmap->stats.external);
}
}
UNLOCK_PVH(pai);
if (pmap != kernel_pmap) {
pmap_ledger_credit(pmap, task_ledgers.phys_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
if (is_internal) {
pmap_ledger_credit(pmap, task_ledgers.internal, pt_attr_page_size(pt_attr) * PAGE_RATIO);
if (is_altacct) {
pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, pt_attr_page_size(pt_attr) * PAGE_RATIO);
} else {
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
}
}
OSAddAtomic(1, (SInt32 *) &pmap->stats.resident_count);
if (pmap->stats.resident_count > pmap->stats.resident_max) {
pmap->stats.resident_max = pmap->stats.resident_count;
}
} else {
if (prot & VM_PROT_EXECUTE) {
kr = KERN_FAILURE;
goto Pmap_enter_cleanup;
}
wimg_bits = pmap_cache_attributes(pn);
if ((flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT))) {
wimg_bits = (wimg_bits & (~VM_WIMG_MASK)) | (flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT));
}
pte |= pmap_get_pt_ops(pmap)->wimg_to_pte(wimg_bits);
#if XNU_MONITOR
if ((wimg_bits & PP_ATTR_MONITOR) && !pmap_ppl_disable) {
uint64_t xprr_perm = pte_to_xprr_perm(pte);
switch (xprr_perm) {
case XPRR_KERN_RO_PERM:
break;
case XPRR_KERN_RW_PERM:
pte &= ~ARM_PTE_XPRR_MASK;
pte |= xprr_perm_to_pte(XPRR_PPL_RW_PERM);
break;
default:
panic("Unsupported xPRR perm %llu for pte 0x%llx", xprr_perm, (uint64_t)pte);
}
}
#endif
pmap_enter_pte(pmap, pte_p, pte, v);
}
goto Pmap_enter_return;
Pmap_enter_cleanup:
if (refcnt != NULL) {
assert(refcnt_updated);
if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= 0) {
panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
}
}
Pmap_enter_return:
#if CONFIG_PGTRACE
if (pgtrace_enabled) {
pmap_pgtrace_enter_clone(pmap, v + ARM_PGBYTES, 0, 0);
}
#endif
if (pve_p != PV_ENTRY_NULL) {
pv_free_entry(pve_p);
}
if (wiredcnt_updated && (OSAddAtomic16(-1, (volatile int16_t*)wiredcnt) <= 0)) {
panic("pmap_enter(): over-unwire of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p);
}
pmap_unlock(pmap);
return kr;
}
kern_return_t
pmap_enter_options_addr(
pmap_t pmap,
vm_map_address_t v,
pmap_paddr_t pa,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options,
__unused void *arg)
{
kern_return_t kr = KERN_FAILURE;
PMAP_TRACE(2, PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v), pa, prot);
#if XNU_MONITOR
while ((kr = pmap_enter_options_ppl(pmap, v, pa, prot, fault_type, flags, wired, options | PMAP_OPTIONS_NOWAIT)) == KERN_RESOURCE_SHORTAGE) {
pmap_alloc_page_for_ppl((options & PMAP_OPTIONS_NOWAIT) ? PMAP_PAGES_ALLOCATE_NOWAIT : 0);
if (options & PMAP_OPTIONS_NOWAIT) {
break;
}
}
pmap_ledger_check_balance(pmap);
#else
kr = pmap_enter_options_internal(pmap, v, pa, prot, fault_type, flags, wired, options);
#endif
PMAP_TRACE(2, PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr);
return kr;
}
kern_return_t
pmap_enter_options(
pmap_t pmap,
vm_map_address_t v,
ppnum_t pn,
vm_prot_t prot,
vm_prot_t fault_type,
unsigned int flags,
boolean_t wired,
unsigned int options,
__unused void *arg)
{
return pmap_enter_options_addr(pmap, v, ((pmap_paddr_t)pn) << PAGE_SHIFT, prot, fault_type, flags, wired, options, arg);
}
MARK_AS_PMAP_TEXT static void
pmap_change_wiring_internal(
pmap_t pmap,
vm_map_address_t v,
boolean_t wired)
{
pt_entry_t *pte_p;
pmap_paddr_t pa;
VALIDATE_PMAP(pmap);
pmap_lock(pmap);
const pt_attr_t * pt_attr = pmap_get_pt_attr(pmap);
pte_p = pmap_pte(pmap, v);
if (pte_p == PT_ENTRY_NULL) {
if (!wired) {
goto pmap_change_wiring_return;
} else {
panic("%s: Attempt to wire nonexistent PTE for pmap %p", __func__, pmap);
}
}
pa = pte_to_pa(*((volatile pt_entry_t*)pte_p));
while (pa_valid(pa)) {
pmap_paddr_t new_pa;
LOCK_PVH((int)pa_index(pa));
new_pa = pte_to_pa(*((volatile pt_entry_t*)pte_p));
if (pa == new_pa) {
break;
}
UNLOCK_PVH((int)pa_index(pa));
pa = new_pa;
}
if ((*pte_p == ARM_PTE_EMPTY) || (ARM_PTE_IS_COMPRESSED(*pte_p, pte_p))) {
if (!wired) {
goto pmap_change_wiring_cleanup;
} else {
panic("%s: Attempt to wire empty/compressed PTE %p (=0x%llx) for pmap %p",
__func__, pte_p, (uint64_t)*pte_p, pmap);
}
}
assertf((*pte_p & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE, "invalid pte %p (=0x%llx)", pte_p, (uint64_t)*pte_p);
if (wired != pte_is_wired(*pte_p)) {
pte_set_wired(pmap, pte_p, wired);
if (pmap != kernel_pmap) {
if (wired) {
OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
pmap_ledger_credit(pmap, task_ledgers.wired_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
} else if (!wired) {
__assert_only int32_t orig_wired = OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
PMAP_STATS_ASSERTF(orig_wired > 0, pmap, "stats.wired_count %d", orig_wired);
pmap_ledger_debit(pmap, task_ledgers.wired_mem, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
}
}
pmap_change_wiring_cleanup:
if (pa_valid(pa)) {
UNLOCK_PVH((int)pa_index(pa));
}
pmap_change_wiring_return:
pmap_unlock(pmap);
}
void
pmap_change_wiring(
pmap_t pmap,
vm_map_address_t v,
boolean_t wired)
{
#if XNU_MONITOR
pmap_change_wiring_ppl(pmap, v, wired);
pmap_ledger_check_balance(pmap);
#else
pmap_change_wiring_internal(pmap, v, wired);
#endif
}
MARK_AS_PMAP_TEXT static pmap_paddr_t
pmap_find_pa_internal(
pmap_t pmap,
addr64_t va)
{
pmap_paddr_t pa = 0;
VALIDATE_PMAP(pmap);
if (pmap != kernel_pmap) {
pmap_lock_ro(pmap);
}
pa = pmap_vtophys(pmap, va);
if (pmap != kernel_pmap) {
pmap_unlock_ro(pmap);
}
return pa;
}
pmap_paddr_t
pmap_find_pa_nofault(pmap_t pmap, addr64_t va)
{
pmap_paddr_t pa = 0;
if (pmap == kernel_pmap) {
pa = mmu_kvtop(va);
} else if ((current_thread()->map) && (pmap == vm_map_pmap(current_thread()->map))) {
pa = mmu_uvtop(va);
}
return pa;
}
pmap_paddr_t
pmap_find_pa(
pmap_t pmap,
addr64_t va)
{
pmap_paddr_t pa = pmap_find_pa_nofault(pmap, va);
if (pa != 0) {
return pa;
}
if (not_in_kdp) {
#if XNU_MONITOR
return pmap_find_pa_ppl(pmap, va);
#else
return pmap_find_pa_internal(pmap, va);
#endif
} else {
return pmap_vtophys(pmap, va);
}
}
ppnum_t
pmap_find_phys_nofault(
pmap_t pmap,
addr64_t va)
{
ppnum_t ppn;
ppn = atop(pmap_find_pa_nofault(pmap, va));
return ppn;
}
ppnum_t
pmap_find_phys(
pmap_t pmap,
addr64_t va)
{
ppnum_t ppn;
ppn = atop(pmap_find_pa(pmap, va));
return ppn;
}
pmap_paddr_t
kvtophys(
vm_offset_t va)
{
pmap_paddr_t pa;
pa = mmu_kvtop(va);
if (pa) {
return pa;
}
return pmap_vtophys(kernel_pmap, va);
}
pmap_paddr_t
pmap_vtophys(
pmap_t pmap,
addr64_t va)
{
if ((va < pmap->min) || (va >= pmap->max)) {
return 0;
}
#if (__ARM_VMSA__ == 7)
tt_entry_t *tte_p, tte;
pt_entry_t *pte_p;
pmap_paddr_t pa;
tte_p = pmap_tte(pmap, va);
if (tte_p == (tt_entry_t *) NULL) {
return (pmap_paddr_t) 0;
}
tte = *tte_p;
if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(tte) + pte_index(pmap, pt_attr, va);
pa = pte_to_pa(*pte_p) | (va & ARM_PGMASK);
#if DEVELOPMENT || DEBUG
if (atop(pa) != 0 &&
ARM_PTE_IS_COMPRESSED(*pte_p, pte_p)) {
panic("pmap_vtophys(%p,0x%llx): compressed pte_p=%p 0x%llx with ppn=0x%x\n",
pmap, va, pte_p, (uint64_t) (*pte_p), atop(pa));
}
#endif
} else if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
if ((tte & ARM_TTE_BLOCK_SUPER) == ARM_TTE_BLOCK_SUPER) {
pa = suptte_to_pa(tte) | (va & ARM_TT_L1_SUPER_OFFMASK);
} else {
pa = sectte_to_pa(tte) | (va & ARM_TT_L1_BLOCK_OFFMASK);
}
} else {
pa = 0;
}
#else
tt_entry_t * ttp = NULL;
tt_entry_t * ttep = NULL;
tt_entry_t tte = ARM_TTE_EMPTY;
pmap_paddr_t pa = 0;
unsigned int cur_level;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
ttp = pmap->tte;
for (cur_level = pt_attr_root_level(pt_attr); cur_level <= pt_attr_leaf_level(pt_attr); cur_level++) {
ttep = &ttp[ttn_index(pmap, pt_attr, va, cur_level)];
tte = *ttep;
const uint64_t valid_mask = pt_attr->pta_level_info[cur_level].valid_mask;
const uint64_t type_mask = pt_attr->pta_level_info[cur_level].type_mask;
const uint64_t type_block = pt_attr->pta_level_info[cur_level].type_block;
const uint64_t offmask = pt_attr->pta_level_info[cur_level].offmask;
if ((tte & valid_mask) != valid_mask) {
return (pmap_paddr_t) 0;
}
if ((tte & type_mask) == type_block) {
pa = ((tte & ARM_TTE_PA_MASK & ~offmask) | (va & offmask));
break;
}
ttp = (tt_entry_t*)phystokv(tte & ARM_TTE_TABLE_MASK);
}
#endif
return pa;
}
void
pmap_init_pte_page(
pmap_t pmap,
pt_entry_t *pte_p,
vm_offset_t va,
unsigned int ttlevel,
boolean_t alloc_ptd)
{
pt_desc_t *ptdp = NULL;
vm_offset_t *pvh;
pvh = (vm_offset_t *)(pai_to_pvh(pa_index(ml_static_vtop((vm_offset_t)pte_p))));
if (pvh_test_type(pvh, PVH_TYPE_NULL)) {
if (alloc_ptd) {
ptdp = ptd_alloc(pmap);
if (ptdp == NULL) {
panic("%s: unable to allocate PTD", __func__);
}
pvh_update_head_unlocked(pvh, ptdp, PVH_TYPE_PTDP);
} else {
panic("pmap_init_pte_page(): pte_p %p", pte_p);
}
} else if (pvh_test_type(pvh, PVH_TYPE_PTDP)) {
ptdp = (pt_desc_t*)(pvh_list(pvh));
} else {
panic("pmap_init_pte_page(): invalid PVH type for pte_p %p", pte_p);
}
__builtin_arm_dmb(DMB_ISHST);
ptd_init(ptdp, pmap, va, ttlevel, pte_p);
}
static kern_return_t
pmap_expand(
pmap_t pmap,
vm_map_address_t v,
unsigned int options,
unsigned int level)
{
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
#if (__ARM_VMSA__ == 7)
vm_offset_t pa;
tt_entry_t *tte_p;
tt_entry_t *tt_p;
unsigned int i;
#if DEVELOPMENT || DEBUG
i = tte_index(pmap, pt_attr, v);
if (i >= pmap->tte_index_max) {
panic("%s: index out of range, index=%u, max=%u, "
"pmap=%p, addr=%p, options=%u, level=%u",
__func__, i, pmap->tte_index_max,
pmap, (void *)v, options, level);
}
#endif
if (level == 1) {
return KERN_SUCCESS;
}
{
tt_entry_t *tte_next_p;
pmap_lock(pmap);
pa = 0;
if (pmap_pte(pmap, v) != PT_ENTRY_NULL) {
pmap_unlock(pmap);
return KERN_SUCCESS;
}
tte_p = &pmap->tte[ttenum(v & ~ARM_TT_L1_PT_OFFMASK)];
for (i = 0, tte_next_p = tte_p; i < 4; i++) {
if (tte_to_pa(*tte_next_p)) {
pa = tte_to_pa(*tte_next_p);
break;
}
tte_next_p++;
}
pa = pa & ~PAGE_MASK;
if (pa) {
tte_p = &pmap->tte[ttenum(v)];
*tte_p = pa_to_tte(pa) | (((v >> ARM_TT_L1_SHIFT) & 0x3) << 10) | ARM_TTE_TYPE_TABLE;
FLUSH_PTE(tte_p);
PMAP_TRACE(5, PMAP_CODE(PMAP__TTE), VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v & ~ARM_TT_L1_OFFMASK),
VM_KERNEL_ADDRHIDE((v & ~ARM_TT_L1_OFFMASK) + ARM_TT_L1_SIZE), *tte_p);
pmap_unlock(pmap);
return KERN_SUCCESS;
}
pmap_unlock(pmap);
}
v = v & ~ARM_TT_L1_PT_OFFMASK;
while (pmap_pte(pmap, v) == PT_ENTRY_NULL) {
while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if (options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
VM_PAGE_WAIT();
}
pmap_lock(pmap);
if (pmap_pte(pmap, v) == PT_ENTRY_NULL) {
tt_entry_t *tte_next_p;
pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE);
pa = kvtophys((vm_offset_t)tt_p);
tte_p = &pmap->tte[ttenum(v)];
for (i = 0, tte_next_p = tte_p; i < 4; i++) {
*tte_next_p = pa_to_tte(pa) | ARM_TTE_TYPE_TABLE;
PMAP_TRACE(5, PMAP_CODE(PMAP__TTE), VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE((v & ~ARM_TT_L1_PT_OFFMASK) + (i * ARM_TT_L1_SIZE)),
VM_KERNEL_ADDRHIDE((v & ~ARM_TT_L1_PT_OFFMASK) + ((i + 1) * ARM_TT_L1_SIZE)), *tte_p);
tte_next_p++;
pa = pa + 0x400;
}
FLUSH_PTE_RANGE(tte_p, tte_p + 4);
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
}
pmap_unlock(pmap);
if (tt_p != (tt_entry_t *)NULL) {
pmap_tt_deallocate(pmap, tt_p, PMAP_TT_L2_LEVEL);
tt_p = (tt_entry_t *)NULL;
}
}
return KERN_SUCCESS;
#else
pmap_paddr_t pa;
unsigned int ttlevel = pt_attr_root_level(pt_attr);
tt_entry_t *tte_p;
tt_entry_t *tt_p;
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
for (; ttlevel < level; ttlevel++) {
pmap_lock_ro(pmap);
if (pmap_ttne(pmap, ttlevel + 1, v) == PT_ENTRY_NULL) {
pmap_unlock_ro(pmap);
while (pmap_tt_allocate(pmap, &tt_p, ttlevel + 1, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) {
if (options & PMAP_OPTIONS_NOWAIT) {
return KERN_RESOURCE_SHORTAGE;
}
#if XNU_MONITOR
panic("%s: failed to allocate tt, "
"pmap=%p, v=%p, options=0x%x, level=%u",
__FUNCTION__,
pmap, (void *)v, options, level);
#else
VM_PAGE_WAIT();
#endif
}
pmap_lock(pmap);
if ((pmap_ttne(pmap, ttlevel + 1, v) == PT_ENTRY_NULL)) {
pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, ttlevel + 1, FALSE);
pa = kvtophys((vm_offset_t)tt_p);
tte_p = pmap_ttne(pmap, ttlevel, v);
*tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID;
PMAP_TRACE(4 + ttlevel, PMAP_CODE(PMAP__TTE), VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v & ~pt_attr_ln_offmask(pt_attr, ttlevel)),
VM_KERNEL_ADDRHIDE((v & ~pt_attr_ln_offmask(pt_attr, ttlevel)) + pt_attr_ln_size(pt_attr, ttlevel)), *tte_p);
pa = 0x0ULL;
tt_p = (tt_entry_t *)NULL;
}
pmap_unlock(pmap);
} else {
pmap_unlock_ro(pmap);
}
if (tt_p != (tt_entry_t *)NULL) {
pmap_tt_deallocate(pmap, tt_p, ttlevel + 1);
tt_p = (tt_entry_t *)NULL;
}
}
return KERN_SUCCESS;
#endif
}
void
pmap_collect(pmap_t pmap)
{
if (pmap == PMAP_NULL) {
return;
}
#if 0
pmap_lock(pmap);
if ((pmap->nested == FALSE) && (pmap != kernel_pmap)) {
}
pmap_unlock(pmap);
#endif
return;
}
void
pmap_gc(
void)
{
#if XNU_MONITOR
#endif
#if !XNU_MONITOR
pmap_t pmap, pmap_next;
boolean_t gc_wait;
if (pmap_gc_allowed &&
(pmap_gc_allowed_by_time_throttle ||
pmap_gc_forced)) {
pmap_gc_forced = FALSE;
pmap_gc_allowed_by_time_throttle = FALSE;
pmap_simple_lock(&pmaps_lock);
pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&map_pmap_list));
while (!queue_end(&map_pmap_list, (queue_entry_t)pmap)) {
if (!(pmap->gc_status & PMAP_GC_INFLIGHT)) {
pmap->gc_status |= PMAP_GC_INFLIGHT;
}
pmap_simple_unlock(&pmaps_lock);
pmap_collect(pmap);
pmap_simple_lock(&pmaps_lock);
gc_wait = (pmap->gc_status & PMAP_GC_WAIT);
pmap->gc_status &= ~(PMAP_GC_INFLIGHT | PMAP_GC_WAIT);
pmap_next = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&pmap->pmaps));
if (gc_wait) {
if (!queue_end(&map_pmap_list, (queue_entry_t)pmap_next)) {
pmap_next->gc_status |= PMAP_GC_INFLIGHT;
}
pmap_simple_unlock(&pmaps_lock);
thread_wakeup((event_t) &pmap->gc_status);
pmap_simple_lock(&pmaps_lock);
}
pmap = pmap_next;
}
pmap_simple_unlock(&pmaps_lock);
}
#endif
}
uint64_t
pmap_release_pages_fast(void)
{
#if XNU_MONITOR
return pmap_release_ppl_pages_to_kernel();
#else
return 0;
#endif
}
void
compute_pmap_gc_throttle(
void *arg __unused)
{
pmap_gc_allowed_by_time_throttle = TRUE;
}
kern_return_t
pmap_attribute_cache_sync(
ppnum_t pp,
vm_size_t size,
__unused vm_machine_attribute_t attribute,
__unused vm_machine_attribute_val_t * value)
{
if (size > PAGE_SIZE) {
panic("pmap_attribute_cache_sync size: 0x%llx\n", (uint64_t)size);
} else {
cache_sync_page(pp);
}
return KERN_SUCCESS;
}
void
pmap_sync_page_data_phys(
ppnum_t pp)
{
cache_sync_page(pp);
}
void
pmap_sync_page_attributes_phys(
ppnum_t pp)
{
flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE);
}
#if CONFIG_COREDUMP
boolean_t
coredumpok(
vm_map_t map,
mach_vm_offset_t va)
{
pt_entry_t *pte_p;
pt_entry_t spte;
pte_p = pmap_pte(map->pmap, va);
if (0 == pte_p) {
return FALSE;
}
spte = *pte_p;
return (spte & ARM_PTE_ATTRINDXMASK) == ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
}
#endif
void
fillPage(
ppnum_t pn,
unsigned int fill)
{
unsigned int *addr;
int count;
addr = (unsigned int *) phystokv(ptoa(pn));
count = PAGE_SIZE / sizeof(unsigned int);
while (count--) {
*addr++ = fill;
}
}
extern void mapping_set_mod(ppnum_t pn);
void
mapping_set_mod(
ppnum_t pn)
{
pmap_set_modify(pn);
}
extern void mapping_set_ref(ppnum_t pn);
void
mapping_set_ref(
ppnum_t pn)
{
pmap_set_reference(pn);
}
MARK_AS_PMAP_TEXT static void
phys_attribute_clear_with_flush_range(
ppnum_t pn,
unsigned int bits,
int options,
void *arg,
pmap_tlb_flush_range_t *flush_range)
{
pmap_paddr_t pa = ptoa(pn);
vm_prot_t allow_mode = VM_PROT_ALL;
#if XNU_MONITOR
if (__improbable(bits & PP_ATTR_PPL_OWNED_BITS)) {
panic("%s: illegal request, "
"pn=%u, bits=%#x, options=%#x, arg=%p, flush_range=%p",
__FUNCTION__,
pn, bits, options, arg, flush_range);
}
#endif
if ((arg != NULL) || (flush_range != NULL)) {
options = options & ~PMAP_OPTIONS_NOFLUSH;
}
if (__improbable((bits & PP_ATTR_MODIFIED) &&
(options & PMAP_OPTIONS_NOFLUSH))) {
panic("phys_attribute_clear(0x%x,0x%x,0x%x,%p,%p): "
"should not clear 'modified' without flushing TLBs\n",
pn, bits, options, arg, flush_range);
}
assert(pn != vm_page_fictitious_addr);
if (options & PMAP_OPTIONS_CLEAR_WRITE) {
assert(bits == PP_ATTR_MODIFIED);
pmap_page_protect_options_with_flush_range(pn, (VM_PROT_ALL & ~VM_PROT_WRITE), options, flush_range);
pa_clear_bits(pa, bits);
return;
}
if (bits & PP_ATTR_REFERENCED) {
allow_mode &= ~(VM_PROT_READ | VM_PROT_EXECUTE);
}
if (bits & PP_ATTR_MODIFIED) {
allow_mode &= ~VM_PROT_WRITE;
}
if (bits == PP_ATTR_NOENCRYPT) {
pa_clear_bits(pa, bits);
return;
}
if (arm_force_fast_fault_with_flush_range(pn, allow_mode, options, flush_range)) {
pa_clear_bits(pa, bits);
}
}
MARK_AS_PMAP_TEXT static void
phys_attribute_clear_internal(
ppnum_t pn,
unsigned int bits,
int options,
void *arg)
{
phys_attribute_clear_with_flush_range(pn, bits, options, arg, NULL);
}
#if __ARM_RANGE_TLBI__
MARK_AS_PMAP_TEXT static vm_map_address_t
phys_attribute_clear_twig_internal(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
unsigned int bits,
unsigned int options,
pmap_tlb_flush_range_t *flush_range)
{
pmap_assert_locked_r(pmap);
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
assert(end >= start);
assert((end - start) <= pt_attr_twig_size(pt_attr));
const uint64_t pmap_page_size = pt_attr_page_size(pt_attr);
vm_map_address_t va = start;
pt_entry_t *pte_p, *start_pte_p, *end_pte_p, *curr_pte_p;
tt_entry_t *tte_p;
tte_p = pmap_tte(pmap, start);
unsigned int npages = 0;
if (tte_p == (tt_entry_t *) NULL) {
return end;
}
if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
start_pte_p = &pte_p[pte_index(pmap, pt_attr, start)];
end_pte_p = start_pte_p + ((end - start) >> pt_attr_leaf_shift(pt_attr));
assert(end_pte_p >= start_pte_p);
for (curr_pte_p = start_pte_p; curr_pte_p < end_pte_p; curr_pte_p++, va += pmap_page_size) {
if (__improbable(npages++ && pmap_pending_preemption())) {
return va;
}
pmap_paddr_t pa = pte_to_pa(*((volatile pt_entry_t*)curr_pte_p));
if (pa_valid(pa)) {
ppnum_t pn = (ppnum_t) atop(pa);
phys_attribute_clear_with_flush_range(pn, bits, options, NULL, flush_range);
}
}
}
return end;
}
MARK_AS_PMAP_TEXT static vm_map_address_t
phys_attribute_clear_range_internal(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
unsigned int bits,
unsigned int options)
{
if (__improbable(end < start)) {
panic("%s: invalid address range %p, %p", __func__, (void*)start, (void*)end);
}
VALIDATE_PMAP(pmap);
vm_map_address_t va = start;
pmap_tlb_flush_range_t flush_range = {
.ptfr_pmap = pmap,
.ptfr_start = start,
.ptfr_end = end,
.ptfr_flush_needed = false
};
pmap_lock_ro(pmap);
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
while (va < end) {
vm_map_address_t curr_end;
curr_end = ((va + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr));
if (curr_end > end) {
curr_end = end;
}
va = phys_attribute_clear_twig_internal(pmap, va, curr_end, bits, options, &flush_range);
if ((va < curr_end) || pmap_pending_preemption()) {
break;
}
}
pmap_unlock_ro(pmap);
if (flush_range.ptfr_flush_needed) {
flush_range.ptfr_end = va;
pmap_get_pt_ops(pmap)->flush_tlb_region_async(
flush_range.ptfr_start,
flush_range.ptfr_end - flush_range.ptfr_start,
flush_range.ptfr_pmap);
sync_tlb_flush();
}
return va;
}
static void
phys_attribute_clear_range(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
unsigned int bits,
unsigned int options)
{
assert(get_preemption_level() == 0);
PMAP_TRACE(3, PMAP_CODE(PMAP__ATTRIBUTE_CLEAR_RANGE) | DBG_FUNC_START, bits);
while (start < end) {
#if XNU_MONITOR
start = phys_attribute_clear_range_ppl(pmap, start, end, bits, options);
#else
start = phys_attribute_clear_range_internal(pmap, start, end, bits, options);
#endif
}
PMAP_TRACE(3, PMAP_CODE(PMAP__ATTRIBUTE_CLEAR_RANGE) | DBG_FUNC_END);
}
#endif
static void
phys_attribute_clear(
ppnum_t pn,
unsigned int bits,
int options,
void *arg)
{
PMAP_TRACE(3, PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits);
#if XNU_MONITOR
phys_attribute_clear_ppl(pn, bits, options, arg);
#else
phys_attribute_clear_internal(pn, bits, options, arg);
#endif
PMAP_TRACE(3, PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END);
}
MARK_AS_PMAP_TEXT static void
phys_attribute_set_internal(
ppnum_t pn,
unsigned int bits)
{
pmap_paddr_t pa = ptoa(pn);
assert(pn != vm_page_fictitious_addr);
#if XNU_MONITOR
if (bits & PP_ATTR_PPL_OWNED_BITS) {
panic("%s: illegal request, "
"pn=%u, bits=%#x",
__FUNCTION__,
pn, bits);
}
#endif
pa_set_bits(pa, (uint16_t)bits);
return;
}
static void
phys_attribute_set(
ppnum_t pn,
unsigned int bits)
{
#if XNU_MONITOR
phys_attribute_set_ppl(pn, bits);
#else
phys_attribute_set_internal(pn, bits);
#endif
}
static boolean_t
phys_attribute_test(
ppnum_t pn,
unsigned int bits)
{
pmap_paddr_t pa = ptoa(pn);
assert(pn != vm_page_fictitious_addr);
return pa_test_bits(pa, bits);
}
void
pmap_set_modify(ppnum_t pn)
{
phys_attribute_set(pn, PP_ATTR_MODIFIED);
}
void
pmap_clear_modify(
ppnum_t pn)
{
phys_attribute_clear(pn, PP_ATTR_MODIFIED, 0, NULL);
}
boolean_t
pmap_is_modified(
ppnum_t pn)
{
return phys_attribute_test(pn, PP_ATTR_MODIFIED);
}
static void
pmap_set_reference(
ppnum_t pn)
{
phys_attribute_set(pn, PP_ATTR_REFERENCED);
}
void
pmap_clear_reference(
ppnum_t pn)
{
phys_attribute_clear(pn, PP_ATTR_REFERENCED, 0, NULL);
}
boolean_t
pmap_is_referenced(
ppnum_t pn)
{
return phys_attribute_test(pn, PP_ATTR_REFERENCED);
}
unsigned int
pmap_get_refmod(
ppnum_t pn)
{
return ((phys_attribute_test(pn, PP_ATTR_MODIFIED)) ? VM_MEM_MODIFIED : 0)
| ((phys_attribute_test(pn, PP_ATTR_REFERENCED)) ? VM_MEM_REFERENCED : 0);
}
static inline unsigned int
pmap_clear_refmod_mask_to_modified_bits(const unsigned int mask)
{
return ((mask & VM_MEM_MODIFIED) ? PP_ATTR_MODIFIED : 0) |
((mask & VM_MEM_REFERENCED) ? PP_ATTR_REFERENCED : 0);
}
void
pmap_clear_refmod_options(
ppnum_t pn,
unsigned int mask,
unsigned int options,
void *arg)
{
unsigned int bits;
bits = pmap_clear_refmod_mask_to_modified_bits(mask);
phys_attribute_clear(pn, bits, options, arg);
}
bool
pmap_clear_refmod_range_options(
pmap_t pmap __unused,
vm_map_address_t start __unused,
vm_map_address_t end __unused,
unsigned int mask __unused,
unsigned int options __unused)
{
#if __ARM_RANGE_TLBI__
unsigned int bits;
bits = pmap_clear_refmod_mask_to_modified_bits(mask);
phys_attribute_clear_range(pmap, start, end, bits, options);
return true;
#else
#pragma unused(pmap, start, end, mask, options)
return false;
#endif
}
void
pmap_clear_refmod(
ppnum_t pn,
unsigned int mask)
{
pmap_clear_refmod_options(pn, mask, 0, NULL);
}
unsigned int
pmap_disconnect_options(
ppnum_t pn,
unsigned int options,
void *arg)
{
if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED)) {
if (pmap_is_modified(pn)) {
options |= PMAP_OPTIONS_COMPRESSOR;
} else {
}
}
pmap_page_protect_options(pn, 0, options, arg);
return pmap_get_refmod(pn);
}
unsigned int
pmap_disconnect(
ppnum_t pn)
{
pmap_page_protect(pn, 0);
return pmap_get_refmod(pn);
}
boolean_t
pmap_has_managed_page(ppnum_t first, ppnum_t last)
{
if (ptoa(first) >= vm_last_phys) {
return FALSE;
}
if (ptoa(last) < vm_first_phys) {
return FALSE;
}
return TRUE;
}
boolean_t
pmap_is_noencrypt(
ppnum_t pn)
{
#if DEVELOPMENT || DEBUG
boolean_t result = FALSE;
if (!pa_valid(ptoa(pn))) {
return FALSE;
}
result = (phys_attribute_test(pn, PP_ATTR_NOENCRYPT));
return result;
#else
#pragma unused(pn)
return FALSE;
#endif
}
void
pmap_set_noencrypt(
ppnum_t pn)
{
#if DEVELOPMENT || DEBUG
if (!pa_valid(ptoa(pn))) {
return;
}
phys_attribute_set(pn, PP_ATTR_NOENCRYPT);
#else
#pragma unused(pn)
#endif
}
void
pmap_clear_noencrypt(
ppnum_t pn)
{
#if DEVELOPMENT || DEBUG
if (!pa_valid(ptoa(pn))) {
return;
}
phys_attribute_clear(pn, PP_ATTR_NOENCRYPT, 0, NULL);
#else
#pragma unused(pn)
#endif
}
#if XNU_MONITOR
boolean_t
pmap_is_monitor(ppnum_t pn)
{
assert(pa_valid(ptoa(pn)));
return phys_attribute_test(pn, PP_ATTR_MONITOR);
}
#endif
void
pmap_lock_phys_page(ppnum_t pn)
{
#if !XNU_MONITOR
int pai;
pmap_paddr_t phys = ptoa(pn);
if (pa_valid(phys)) {
pai = (int)pa_index(phys);
LOCK_PVH(pai);
} else
#else
(void)pn;
#endif
{ simple_lock(&phys_backup_lock, LCK_GRP_NULL);}
}
void
pmap_unlock_phys_page(ppnum_t pn)
{
#if !XNU_MONITOR
int pai;
pmap_paddr_t phys = ptoa(pn);
if (pa_valid(phys)) {
pai = (int)pa_index(phys);
UNLOCK_PVH(pai);
} else
#else
(void)pn;
#endif
{ simple_unlock(&phys_backup_lock);}
}
MARK_AS_PMAP_TEXT static void
pmap_switch_user_ttb_internal(
pmap_t pmap)
{
VALIDATE_PMAP(pmap);
pmap_cpu_data_t *cpu_data_ptr;
cpu_data_ptr = pmap_get_cpu_data();
#if (__ARM_VMSA__ == 7)
cpu_data_ptr->cpu_user_pmap = pmap;
cpu_data_ptr->cpu_user_pmap_stamp = pmap->stamp;
#if MACH_ASSERT && __ARM_USER_PROTECT__
{
unsigned int ttbr0_val, ttbr1_val;
__asm__ volatile ("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val));
__asm__ volatile ("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val));
if (ttbr0_val != ttbr1_val) {
panic("Misaligned ttbr0 %08X\n", ttbr0_val);
}
if (pmap->ttep & 0x1000) {
panic("Misaligned ttbr0 %08X\n", pmap->ttep);
}
}
#endif
#if !__ARM_USER_PROTECT__
set_mmu_ttb(pmap->ttep);
set_context_id(pmap->hw_asid);
#endif
#else
if (pmap != kernel_pmap) {
cpu_data_ptr->cpu_nested_pmap = pmap->nested_pmap;
cpu_data_ptr->cpu_nested_pmap_attr = (cpu_data_ptr->cpu_nested_pmap == NULL) ?
NULL : pmap_get_pt_attr(cpu_data_ptr->cpu_nested_pmap);
cpu_data_ptr->cpu_nested_region_addr = pmap->nested_region_addr;
cpu_data_ptr->cpu_nested_region_size = pmap->nested_region_size;
}
#if __ARM_MIXED_PAGE_SIZE__
if ((pmap != kernel_pmap) && (pmap_get_pt_attr(pmap)->pta_tcr_value != get_tcr())) {
set_tcr(pmap_get_pt_attr(pmap)->pta_tcr_value);
}
#endif
if (pmap != kernel_pmap) {
set_mmu_ttb((pmap->ttep & TTBR_BADDR_MASK) | (((uint64_t)pmap->hw_asid) << TTBR_ASID_SHIFT));
} else if (!pmap_user_ttb_is_clear()) {
pmap_clear_user_ttb_internal();
}
#endif
}
void
pmap_switch_user_ttb(
pmap_t pmap)
{
PMAP_TRACE(1, PMAP_CODE(PMAP__SWITCH_USER_TTB) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(pmap), PMAP_VASID(pmap), pmap->hw_asid);
#if XNU_MONITOR
pmap_switch_user_ttb_ppl(pmap);
#else
pmap_switch_user_ttb_internal(pmap);
#endif
PMAP_TRACE(1, PMAP_CODE(PMAP__SWITCH_USER_TTB) | DBG_FUNC_END);
}
MARK_AS_PMAP_TEXT static void
pmap_clear_user_ttb_internal(void)
{
#if (__ARM_VMSA__ > 7)
set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK);
#else
set_mmu_ttb(kernel_pmap->ttep);
#endif
}
void
pmap_clear_user_ttb(void)
{
PMAP_TRACE(3, PMAP_CODE(PMAP__CLEAR_USER_TTB) | DBG_FUNC_START, NULL, 0, 0);
#if XNU_MONITOR
pmap_clear_user_ttb_ppl();
#else
pmap_clear_user_ttb_internal();
#endif
PMAP_TRACE(3, PMAP_CODE(PMAP__CLEAR_USER_TTB) | DBG_FUNC_END);
}
MARK_AS_PMAP_TEXT static boolean_t
arm_force_fast_fault_with_flush_range(
ppnum_t ppnum,
vm_prot_t allow_mode,
int options,
pmap_tlb_flush_range_t *flush_range)
{
pmap_paddr_t phys = ptoa(ppnum);
pv_entry_t *pve_p;
pt_entry_t *pte_p;
int pai;
boolean_t result;
pv_entry_t **pv_h;
boolean_t is_reusable, is_internal;
boolean_t tlb_flush_needed = FALSE;
boolean_t ref_fault;
boolean_t mod_fault;
boolean_t clear_write_fault = FALSE;
boolean_t ref_aliases_mod = FALSE;
bool mustsynch = ((options & PMAP_OPTIONS_FF_LOCKED) == 0);
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return FALSE;
}
result = TRUE;
ref_fault = FALSE;
mod_fault = FALSE;
pai = (int)pa_index(phys);
if (__probable(mustsynch)) {
LOCK_PVH(pai);
}
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
}
is_reusable = IS_REUSABLE_PAGE(pai);
is_internal = IS_INTERNAL_PAGE(pai);
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pt_entry_t spte;
pt_entry_t tmplate;
pmap_t pmap;
boolean_t update_pte;
if (pve_p != PV_ENTRY_NULL) {
pte_p = pve_get_ptep(pve_p);
}
if (pte_p == PT_ENTRY_NULL) {
panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum);
}
#ifdef PVH_FLAG_IOMMU
if ((vm_offset_t)pte_p & PVH_FLAG_IOMMU) {
goto fff_skip_pve;
}
#endif
if (*pte_p == ARM_PTE_EMPTY) {
panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
}
if (ARM_PTE_IS_COMPRESSED(*pte_p, pte_p)) {
panic("pte is COMPRESSED: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
}
pmap = ptep_get_pmap(pte_p);
const pt_attr_t * pt_attr = pmap_get_pt_attr(pmap);
va = ptep_get_va(pte_p);
assert(va >= pmap->min && va < pmap->max);
if (IS_ALTACCT_PAGE(pai, pve_p)) {
} else if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
is_reusable &&
is_internal &&
pmap != kernel_pmap) {
__assert_only int32_t orig_reusable = OSAddAtomic(-1, &pmap->stats.reusable);
PMAP_STATS_ASSERTF(orig_reusable > 0, pmap, "stats.reusable %d", orig_reusable);
__assert_only int32_t orig_internal = OSAddAtomic(+1, &pmap->stats.internal);
PMAP_STATS_PEAK(pmap->stats.internal);
PMAP_STATS_ASSERTF(orig_internal >= 0, pmap, "stats.internal %d", orig_internal);
pmap_ledger_credit(pmap, task_ledgers.internal, pt_attr_page_size(pt_attr) * PAGE_RATIO);
assert(!IS_ALTACCT_PAGE(pai, pve_p));
assert(IS_INTERNAL_PAGE(pai));
pmap_ledger_credit(pmap, task_ledgers.phys_footprint, pt_attr_page_size(pt_attr) * PAGE_RATIO);
clear_write_fault = TRUE;
} else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
!is_reusable &&
is_internal &&
pmap != kernel_pmap) {
__assert_only int32_t orig_reusable = OSAddAtomic(+1, &pmap->stats.reusable);
PMAP_STATS_PEAK(pmap->stats.reusable);
PMAP_STATS_ASSERTF(orig_reusable >= 0, pmap, "stats.reusable %d", orig_reusable);
__assert_only int32_t orig_internal = OSAddAtomic(-1, &pmap->stats.internal);
PMAP_STATS_ASSERTF(orig_internal > 0, pmap, "stats.internal %d", orig_internal);
pmap_ledger_debit(pmap, task_ledgers.internal, pt_attr_page_size(pt_attr) * PAGE_RATIO);
assert(!IS_ALTACCT_PAGE(pai, pve_p));
assert(IS_INTERNAL_PAGE(pai));
pmap_ledger_debit(pmap, task_ledgers.phys_footprint, pt_attr_page_size(pt_attr) * PAGE_RATIO);
}
bool wiredskip = pte_is_wired(*pte_p) &&
((options & PMAP_OPTIONS_FF_WIRED) == 0);
if (wiredskip) {
result = FALSE;
goto fff_skip_pve;
}
spte = *pte_p;
tmplate = spte;
update_pte = FALSE;
if ((allow_mode & VM_PROT_READ) != VM_PROT_READ) {
tmplate = tmplate & ~ARM_PTE_AF;
update_pte = TRUE;
ref_fault = TRUE;
}
if ((allow_mode & VM_PROT_WRITE) != VM_PROT_WRITE) {
if (pmap == kernel_pmap) {
if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWNA)) {
tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA));
pte_set_was_writeable(tmplate, true);
update_pte = TRUE;
mod_fault = TRUE;
}
} else {
if ((tmplate & ARM_PTE_APMASK) == pt_attr_leaf_rw(pt_attr)) {
tmplate = ((tmplate & ~ARM_PTE_APMASK) | pt_attr_leaf_ro(pt_attr));
pte_set_was_writeable(tmplate, true);
update_pte = TRUE;
mod_fault = TRUE;
}
}
}
#if MACH_ASSERT && XNU_MONITOR
if (is_pte_xprr_protected(pmap, spte)) {
if (pte_to_xprr_perm(spte) != pte_to_xprr_perm(tmplate)) {
panic("%s: attempted to mutate an xPRR mapping pte_p=%p, pmap=%p, pv_h=%p, pve_p=%p, pte=0x%llx, tmplate=0x%llx, va=0x%llx, "
"ppnum=0x%x, options=0x%x, allow_mode=0x%x",
__FUNCTION__, pte_p, pmap, pv_h, pve_p, (unsigned long long)spte, (unsigned long long)tmplate, (unsigned long long)va,
ppnum, options, allow_mode);
}
}
#endif
if (result && update_pte) {
if (options & PMAP_OPTIONS_NOFLUSH) {
WRITE_PTE_FAST(pte_p, tmplate);
} else {
WRITE_PTE_STRONG(pte_p, tmplate);
if (!flush_range ||
((flush_range->ptfr_pmap != pmap) || va >= flush_range->ptfr_end || va < flush_range->ptfr_start)) {
pmap_get_pt_ops(pmap)->flush_tlb_region_async(va,
pt_attr_page_size(pt_attr) * PAGE_RATIO, pmap);
}
tlb_flush_needed = TRUE;
}
}
fff_skip_pve:
pte_p = PT_ENTRY_NULL;
if (pve_p != PV_ENTRY_NULL) {
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
if (clear_write_fault && !ref_aliases_mod) {
arm_clear_fast_fault(ppnum, VM_PROT_WRITE);
}
if (tlb_flush_needed) {
if (flush_range) {
flush_range->ptfr_flush_needed = true;
} else {
sync_tlb_flush();
}
}
if (is_internal) {
if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) &&
is_reusable) {
CLR_REUSABLE_PAGE(pai);
} else if ((options & PMAP_OPTIONS_SET_REUSABLE) &&
!is_reusable) {
SET_REUSABLE_PAGE(pai);
}
}
if (mod_fault) {
SET_MODFAULT_PAGE(pai);
}
if (ref_fault) {
SET_REFFAULT_PAGE(pai);
}
if (__probable(mustsynch)) {
UNLOCK_PVH(pai);
}
return result;
}
MARK_AS_PMAP_TEXT static boolean_t
arm_force_fast_fault_internal(
ppnum_t ppnum,
vm_prot_t allow_mode,
int options)
{
if (__improbable((options & (PMAP_OPTIONS_FF_LOCKED | PMAP_OPTIONS_NOFLUSH)) != 0)) {
panic("arm_force_fast_fault(0x%x, 0x%x, 0x%x): invalid options", ppnum, allow_mode, options);
}
return arm_force_fast_fault_with_flush_range(ppnum, allow_mode, options, NULL);
}
boolean_t
arm_force_fast_fault(
ppnum_t ppnum,
vm_prot_t allow_mode,
int options,
__unused void *arg)
{
pmap_paddr_t phys = ptoa(ppnum);
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(phys)) {
return FALSE;
}
#if XNU_MONITOR
return arm_force_fast_fault_ppl(ppnum, allow_mode, options);
#else
return arm_force_fast_fault_internal(ppnum, allow_mode, options);
#endif
}
MARK_AS_PMAP_TEXT static boolean_t
arm_clear_fast_fault(
ppnum_t ppnum,
vm_prot_t fault_type)
{
pmap_paddr_t pa = ptoa(ppnum);
pv_entry_t *pve_p;
pt_entry_t *pte_p;
int pai;
boolean_t result;
boolean_t tlb_flush_needed = FALSE;
pv_entry_t **pv_h;
assert(ppnum != vm_page_fictitious_addr);
if (!pa_valid(pa)) {
return FALSE;
}
result = FALSE;
pai = (int)pa_index(pa);
ASSERT_PVH_LOCKED(pai);
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
}
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pt_entry_t spte;
pt_entry_t tmplate;
pmap_t pmap;
if (pve_p != PV_ENTRY_NULL) {
pte_p = pve_get_ptep(pve_p);
}
if (pte_p == PT_ENTRY_NULL) {
panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum);
}
#ifdef PVH_FLAG_IOMMU
if ((vm_offset_t)pte_p & PVH_FLAG_IOMMU) {
goto cff_skip_pve;
}
#endif
if (*pte_p == ARM_PTE_EMPTY) {
panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum);
}
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
assert(va >= pmap->min && va < pmap->max);
spte = *pte_p;
tmplate = spte;
if ((fault_type & VM_PROT_WRITE) && (pte_was_writeable(spte))) {
{
if (pmap == kernel_pmap) {
tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA));
} else {
assert(!pmap->nested);
tmplate = ((spte & ~ARM_PTE_APMASK) | pt_attr_leaf_rw(pmap_get_pt_attr(pmap)));
}
}
tmplate |= ARM_PTE_AF;
pte_set_was_writeable(tmplate, false);
pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED);
} else if ((fault_type & VM_PROT_READ) && ((spte & ARM_PTE_AF) != ARM_PTE_AF)) {
tmplate = spte | ARM_PTE_AF;
{
pa_set_bits(pa, PP_ATTR_REFERENCED);
}
}
#if MACH_ASSERT && XNU_MONITOR
if (is_pte_xprr_protected(pmap, spte)) {
if (pte_to_xprr_perm(spte) != pte_to_xprr_perm(tmplate)) {
panic("%s: attempted to mutate an xPRR mapping pte_p=%p, pmap=%p, pv_h=%p, pve_p=%p, pte=0x%llx, tmplate=0x%llx, va=0x%llx, "
"ppnum=0x%x, fault_type=0x%x",
__FUNCTION__, pte_p, pmap, pv_h, pve_p, (unsigned long long)spte, (unsigned long long)tmplate, (unsigned long long)va,
ppnum, fault_type);
}
}
#endif
if (spte != tmplate) {
if (spte != ARM_PTE_TYPE_FAULT) {
WRITE_PTE_STRONG(pte_p, tmplate);
pmap_get_pt_ops(pmap)->flush_tlb_region_async(va,
pt_attr_page_size(pmap_get_pt_attr(pmap)) * PAGE_RATIO, pmap);
tlb_flush_needed = TRUE;
} else {
WRITE_PTE(pte_p, tmplate);
__builtin_arm_isb(ISB_SY);
}
result = TRUE;
}
#ifdef PVH_FLAG_IOMMU
cff_skip_pve:
#endif
pte_p = PT_ENTRY_NULL;
if (pve_p != PV_ENTRY_NULL) {
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
if (tlb_flush_needed) {
sync_tlb_flush();
}
return result;
}
MARK_AS_PMAP_TEXT static kern_return_t
arm_fast_fault_internal(
pmap_t pmap,
vm_map_address_t va,
vm_prot_t fault_type,
__unused bool was_af_fault,
__unused bool from_user)
{
kern_return_t result = KERN_FAILURE;
pt_entry_t *ptep;
pt_entry_t spte = ARM_PTE_TYPE_FAULT;
int pai;
pmap_paddr_t pa;
VALIDATE_PMAP(pmap);
pmap_lock_ro(pmap);
ptep = pmap_pte(pmap, va);
if (ptep != PT_ENTRY_NULL) {
while (true) {
spte = *((volatile pt_entry_t*)ptep);
pa = pte_to_pa(spte);
if ((spte == ARM_PTE_TYPE_FAULT) ||
ARM_PTE_IS_COMPRESSED(spte, ptep)) {
pmap_unlock_ro(pmap);
return result;
}
if (!pa_valid(pa)) {
pmap_unlock_ro(pmap);
#if XNU_MONITOR
if (pmap_cache_attributes((ppnum_t)atop(pa)) & PP_ATTR_MONITOR) {
return KERN_PROTECTION_FAILURE;
} else
#endif
return result;
}
pai = (int)pa_index(pa);
LOCK_PVH(pai);
break;
}
} else {
pmap_unlock_ro(pmap);
return result;
}
if ((result != KERN_SUCCESS) &&
((IS_REFFAULT_PAGE(pai)) || ((fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai)))) {
if (IS_REFFAULT_PAGE(pai)) {
CLR_REFFAULT_PAGE(pai);
}
if ((fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai)) {
CLR_MODFAULT_PAGE(pai);
}
if (arm_clear_fast_fault((ppnum_t)atop(pa), fault_type)) {
result = KERN_SUCCESS;
}
}
if ((result == KERN_FAILURE) && (spte & ARM_PTE_AF)) {
uintptr_t ap_ro, ap_rw, ap_x;
if (pmap == kernel_pmap) {
ap_ro = ARM_PTE_AP(AP_RONA);
ap_rw = ARM_PTE_AP(AP_RWNA);
ap_x = ARM_PTE_NX;
} else {
ap_ro = pt_attr_leaf_ro(pmap_get_pt_attr(pmap));
ap_rw = pt_attr_leaf_rw(pmap_get_pt_attr(pmap));
ap_x = pt_attr_leaf_x(pmap_get_pt_attr(pmap));
}
if (((spte & ARM_PTE_APMASK) == ap_rw) ||
(!(fault_type & VM_PROT_WRITE) && ((spte & ARM_PTE_APMASK) == ap_ro))) {
if (!(fault_type & VM_PROT_EXECUTE) || ((spte & ARM_PTE_XMASK) == ap_x)) {
result = KERN_SUCCESS;
}
}
}
UNLOCK_PVH(pai);
pmap_unlock_ro(pmap);
return result;
}
kern_return_t
arm_fast_fault(
pmap_t pmap,
vm_map_address_t va,
vm_prot_t fault_type,
bool was_af_fault,
__unused bool from_user)
{
kern_return_t result = KERN_FAILURE;
if (va < pmap->min || va >= pmap->max) {
return result;
}
PMAP_TRACE(3, PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(va), fault_type,
from_user);
#if (__ARM_VMSA__ == 7)
if (pmap != kernel_pmap) {
pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data();
pmap_t cur_pmap;
pmap_t cur_user_pmap;
cur_pmap = current_pmap();
cur_user_pmap = cpu_data_ptr->cpu_user_pmap;
if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) {
if (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp) {
pmap_set_pmap(pmap, current_thread());
result = KERN_SUCCESS;
goto done;
}
}
}
#endif
#if XNU_MONITOR
result = arm_fast_fault_ppl(pmap, va, fault_type, was_af_fault, from_user);
#else
result = arm_fast_fault_internal(pmap, va, fault_type, was_af_fault, from_user);
#endif
#if (__ARM_VMSA__ == 7)
done:
#endif
PMAP_TRACE(3, PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_END, result);
return result;
}
void
pmap_copy_page(
ppnum_t psrc,
ppnum_t pdst)
{
bcopy_phys((addr64_t) (ptoa(psrc)),
(addr64_t) (ptoa(pdst)),
PAGE_SIZE);
}
void
pmap_copy_part_page(
ppnum_t psrc,
vm_offset_t src_offset,
ppnum_t pdst,
vm_offset_t dst_offset,
vm_size_t len)
{
bcopy_phys((addr64_t) (ptoa(psrc) + src_offset),
(addr64_t) (ptoa(pdst) + dst_offset),
len);
}
void
pmap_zero_page(
ppnum_t pn)
{
assert(pn != vm_page_fictitious_addr);
bzero_phys((addr64_t) ptoa(pn), PAGE_SIZE);
}
void
pmap_zero_part_page(
ppnum_t pn,
vm_offset_t offset,
vm_size_t len)
{
assert(pn != vm_page_fictitious_addr);
assert(offset + len <= PAGE_SIZE);
bzero_phys((addr64_t) (ptoa(pn) + offset), len);
}
void
pmap_map_globals(
void)
{
pt_entry_t *ptep, pte;
ptep = pmap_pte(kernel_pmap, LOWGLOBAL_ALIAS);
assert(ptep != PT_ENTRY_NULL);
assert(*ptep == ARM_PTE_EMPTY);
pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE;
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
#if (__ARM_VMSA__ > 7)
pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
#else
pte |= ARM_PTE_SH;
#endif
*ptep = pte;
FLUSH_PTE_RANGE(ptep, (ptep + 1));
PMAP_UPDATE_TLBS(kernel_pmap, LOWGLOBAL_ALIAS, LOWGLOBAL_ALIAS + PAGE_SIZE, false);
}
vm_offset_t
pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index)
{
if (__improbable(index >= CPUWINDOWS_MAX)) {
panic("%s: invalid index %u", __func__, index);
}
return (vm_offset_t)(CPUWINDOWS_BASE + (PAGE_SIZE * ((CPUWINDOWS_MAX * cpu_num) + index)));
}
MARK_AS_PMAP_TEXT static unsigned int
pmap_map_cpu_windows_copy_internal(
ppnum_t pn,
vm_prot_t prot,
unsigned int wimg_bits)
{
pt_entry_t *ptep = NULL, pte;
pmap_cpu_data_t *pmap_cpu_data = pmap_get_cpu_data();
unsigned int cpu_num;
unsigned int i;
vm_offset_t cpu_copywindow_vaddr = 0;
bool need_strong_sync = false;
#if XNU_MONITOR
unsigned int cacheattr = (!pa_valid(ptoa(pn) & ARM_PTE_PAGE_MASK) ? pmap_cache_attributes(pn) : 0);
need_strong_sync = ((cacheattr & PMAP_IO_RANGE_STRONG_SYNC) != 0);
#endif
#if XNU_MONITOR
#ifdef __ARM_COHERENT_IO__
if (__improbable(pa_valid(ptoa(pn) & ARM_PTE_PAGE_MASK) && !pmap_ppl_disable)) {
panic("%s: attempted to map a managed page, "
"pn=%u, prot=0x%x, wimg_bits=0x%x",
__FUNCTION__,
pn, prot, wimg_bits);
}
if (__improbable((cacheattr & PP_ATTR_MONITOR) && (prot != VM_PROT_READ) && !pmap_ppl_disable)) {
panic("%s: attempt to map PPL-protected I/O address 0x%llx as writable", __func__, (uint64_t)ptoa(pn));
}
#else
#error CPU copy windows are not properly supported with both the PPL and incoherent IO
#endif
#endif
cpu_num = pmap_cpu_data->cpu_number;
for (i = 0; i < CPUWINDOWS_MAX; i++) {
cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, i);
ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr);
assert(!ARM_PTE_IS_COMPRESSED(*ptep, ptep));
if (*ptep == ARM_PTE_TYPE_FAULT) {
break;
}
}
if (i == CPUWINDOWS_MAX) {
panic("pmap_map_cpu_windows_copy: out of window\n");
}
pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX;
#if __ARM_KERNEL_PROTECT__
pte |= ARM_PTE_NG;
#endif
pte |= wimg_to_pte(wimg_bits);
if (prot & VM_PROT_WRITE) {
pte |= ARM_PTE_AP(AP_RWNA);
} else {
pte |= ARM_PTE_AP(AP_RONA);
}
WRITE_PTE_FAST(ptep, pte);
FLUSH_PTE_STRONG(ptep);
PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE, pmap_cpu_data->copywindow_strong_sync[i]);
pmap_cpu_data->copywindow_strong_sync[i] = need_strong_sync;
return i;
}
unsigned int
pmap_map_cpu_windows_copy(
ppnum_t pn,
vm_prot_t prot,
unsigned int wimg_bits)
{
#if XNU_MONITOR
return pmap_map_cpu_windows_copy_ppl(pn, prot, wimg_bits);
#else
return pmap_map_cpu_windows_copy_internal(pn, prot, wimg_bits);
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_unmap_cpu_windows_copy_internal(
unsigned int index)
{
pt_entry_t *ptep;
unsigned int cpu_num;
vm_offset_t cpu_copywindow_vaddr = 0;
pmap_cpu_data_t *pmap_cpu_data = pmap_get_cpu_data();
cpu_num = pmap_cpu_data->cpu_number;
cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, index);
__builtin_arm_dsb(DSB_SY);
ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr);
WRITE_PTE_STRONG(ptep, ARM_PTE_TYPE_FAULT);
PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE, pmap_cpu_data->copywindow_strong_sync[index]);
}
void
pmap_unmap_cpu_windows_copy(
unsigned int index)
{
#if XNU_MONITOR
return pmap_unmap_cpu_windows_copy_ppl(index);
#else
return pmap_unmap_cpu_windows_copy_internal(index);
#endif
}
#if XNU_MONITOR
MARK_AS_PMAP_TEXT void
pmap_invoke_with_page(
ppnum_t page_number,
void *ctx,
void (*callback)(void *ctx, ppnum_t page_number, const void *page))
{
#pragma unused(page_number, ctx, callback)
}
MARK_AS_PMAP_TEXT void
pmap_hibernate_invoke(void *ctx, void (*callback)(void *ctx, uint64_t addr, uint64_t len))
{
for (unsigned int i = 0; i < num_io_rgns; ++i) {
if (io_attr_table[i].wimg & PMAP_IO_RANGE_NEEDS_HIBERNATING) {
callback(ctx, io_attr_table[i].addr, io_attr_table[i].len);
}
}
}
MARK_AS_PMAP_TEXT void
pmap_set_ppl_hashed_flag(const pmap_paddr_t addr)
{
if (!pa_valid(addr)) {
return;
}
const int pai = (int)pa_index(addr);
if (pp_attr_table[pai] & PP_ATTR_MONITOR) {
pv_entry_t **pv_h = pai_to_pvh(pai);
LOCK_PVH(pai);
pvh_set_flags(pv_h, pvh_get_flags(pv_h) | PVH_FLAG_HASHED);
UNLOCK_PVH(pai);
}
}
MARK_AS_PMAP_TEXT void
pmap_clear_ppl_hashed_flag_all(void)
{
const int last_index = (int)pa_index(vm_last_phys);
pv_entry_t **pv_h = NULL;
for (int pai = 0; pai < last_index; ++pai) {
pv_h = pai_to_pvh(pai);
if ((pvh_get_flags(pv_h) & PVH_FLAG_HASHED) &&
(pp_attr_table[pai] & PP_ATTR_MONITOR)) {
LOCK_PVH(pai);
pvh_set_flags(pv_h, pvh_get_flags(pv_h) & ~PVH_FLAG_HASHED);
UNLOCK_PVH(pai);
}
}
}
MARK_AS_PMAP_TEXT void
pmap_check_ppl_hashed_flag_all(void)
{
const int last_index = (int)pa_index(vm_last_phys);
pv_entry_t **pv_h = NULL;
for (int pai = 0; pai < last_index; ++pai) {
pv_h = pai_to_pvh(pai);
const bool is_pmap_stack = (pai >= (int)pa_index(pmap_stacks_start_pa)) &&
(pai < (int)pa_index(pmap_stacks_end_pa));
if (!is_pmap_stack &&
(pp_attr_table[pai] & PP_ATTR_MONITOR) &&
!(pvh_get_flags(pv_h) & PVH_FLAG_HASHED)) {
panic("Found PPL-owned page that was not hashed into the hibernation image: pai %d", pai);
}
}
}
#endif
MARK_AS_PMAP_TEXT static void
pmap_set_nested_internal(
pmap_t pmap)
{
VALIDATE_PMAP(pmap);
pmap->nested = TRUE;
pmap_get_pt_ops(pmap)->free_id(pmap);
}
void
pmap_set_nested(
pmap_t pmap)
{
#if XNU_MONITOR
pmap_set_nested_ppl(pmap);
#else
pmap_set_nested_internal(pmap);
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_trim_range(
pmap_t pmap,
addr64_t start,
addr64_t end)
{
addr64_t cur;
addr64_t nested_region_start;
addr64_t nested_region_end;
addr64_t adjusted_start;
addr64_t adjusted_end;
addr64_t adjust_offmask;
tt_entry_t * tte_p;
pt_entry_t * pte_p;
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if (__improbable(end < start)) {
panic("%s: invalid address range, "
"pmap=%p, start=%p, end=%p",
__func__,
pmap, (void*)start, (void*)end);
}
nested_region_start = pmap->nested_region_addr;
nested_region_end = nested_region_start + pmap->nested_region_size;
if (__improbable((start < nested_region_start) || (end > nested_region_end))) {
panic("%s: range outside nested region %p-%p, "
"pmap=%p, start=%p, end=%p",
__func__, (void *)nested_region_start, (void *)nested_region_end,
pmap, (void*)start, (void*)end);
}
adjust_offmask = pt_attr_leaf_table_offmask(pt_attr);
adjusted_start = ((start + adjust_offmask) & ~adjust_offmask);
adjusted_end = end & ~adjust_offmask;
for (cur = adjusted_start; (cur < adjusted_end) && (cur >= adjusted_start); cur += pt_attr_twig_size(pt_attr)) {
pmap_lock(pmap);
tte_p = pmap_tte(pmap, cur);
if (tte_p == (tt_entry_t *) NULL) {
goto done;
}
if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
if ((ptep_get_info(pte_p)->refcnt == 0) &&
(pmap != kernel_pmap)) {
if (pmap->nested == TRUE) {
pmap_tte_deallocate(pmap, cur, cur + PAGE_SIZE, false, tte_p, pt_attr_twig_level(pt_attr));
} else {
pmap_tte_remove(pmap, cur, cur + PAGE_SIZE, false, tte_p, pt_attr_twig_level(pt_attr));
}
}
}
done:
pmap_unlock(pmap);
}
#if (__ARM_VMSA__ > 7)
adjusted_start = ((start + pt_attr_ln_offmask(pt_attr, PMAP_TT_L1_LEVEL)) & ~pt_attr_ln_offmask(pt_attr, PMAP_TT_L1_LEVEL));
adjusted_end = end & ~pt_attr_ln_offmask(pt_attr, PMAP_TT_L1_LEVEL);
for (cur = adjusted_start; (cur < adjusted_end) && (cur >= adjusted_start); cur += pt_attr_ln_size(pt_attr, PMAP_TT_L1_LEVEL)) {
pmap_lock(pmap);
bool remove_tt1e = true;
tt_entry_t * tt1e_p = pmap_tt1e(pmap, cur);
tt_entry_t * tt2e_start;
tt_entry_t * tt2e_end;
tt_entry_t * tt2e_p;
tt_entry_t tt1e;
if (tt1e_p == NULL) {
pmap_unlock(pmap);
continue;
}
tt1e = *tt1e_p;
if (tt1e == ARM_TTE_TYPE_FAULT) {
pmap_unlock(pmap);
continue;
}
tt2e_start = &((tt_entry_t*) phystokv(tt1e & ARM_TTE_TABLE_MASK))[0];
tt2e_end = &tt2e_start[pt_attr_page_size(pt_attr) / sizeof(*tt2e_start)];
for (tt2e_p = tt2e_start; tt2e_p < tt2e_end; tt2e_p++) {
if (*tt2e_p != ARM_TTE_TYPE_FAULT) {
remove_tt1e = false;
}
}
if (remove_tt1e) {
pmap_tte_deallocate(pmap, cur, cur + PAGE_SIZE, false, tt1e_p, PMAP_TT_L1_LEVEL);
}
pmap_unlock(pmap);
}
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_trim_internal(
pmap_t grand,
pmap_t subord,
addr64_t vstart,
uint64_t size)
{
addr64_t vend;
addr64_t adjust_offmask;
if (__improbable(os_add_overflow(vstart, size, &vend))) {
panic("%s: grand addr wraps around, "
"grand=%p, subord=%p, vstart=%p, size=%#llx",
__func__, grand, subord, (void*)vstart, size);
}
VALIDATE_PMAP(grand);
VALIDATE_PMAP(subord);
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(grand);
pmap_lock(subord);
if (__improbable(!subord->nested)) {
panic("%s: subord is not nestable, "
"grand=%p, subord=%p, vstart=%p, size=%#llx",
__func__, grand, subord, (void*)vstart, size);
}
if (__improbable(grand->nested)) {
panic("%s: grand is nestable, "
"grand=%p, subord=%p, vstart=%p, size=%#llx",
__func__, grand, subord, (void*)vstart, size);
}
if (__improbable(grand->nested_pmap != subord)) {
panic("%s: grand->nested != subord, "
"grand=%p, subord=%p, vstart=%p, size=%#llx",
__func__, grand, subord, (void*)vstart, size);
}
if (__improbable((size != 0) &&
((vstart < grand->nested_region_addr) || (vend > (grand->nested_region_addr + grand->nested_region_size))))) {
panic("%s: grand range not in nested region, "
"grand=%p, subord=%p, vstart=%p, size=%#llx",
__func__, grand, subord, (void*)vstart, size);
}
if (!grand->nested_has_no_bounds_ref) {
assert(subord->nested_bounds_set);
if (!grand->nested_bounds_set) {
grand->nested_region_true_start = subord->nested_region_true_start;
grand->nested_region_true_end = subord->nested_region_true_end;
grand->nested_bounds_set = true;
}
pmap_unlock(subord);
return;
}
if ((!subord->nested_bounds_set) && size) {
adjust_offmask = pt_attr_leaf_table_offmask(pt_attr);
subord->nested_region_true_start = vstart;
subord->nested_region_true_end = vend;
subord->nested_region_true_start &= ~adjust_offmask;
if (__improbable(os_add_overflow(subord->nested_region_true_end, adjust_offmask, &subord->nested_region_true_end))) {
panic("%s: padded true end wraps around, "
"grand=%p, subord=%p, vstart=%p, size=%#llx",
__func__, grand, subord, (void*)vstart, size);
}
subord->nested_region_true_end &= ~adjust_offmask;
subord->nested_bounds_set = true;
}
if (subord->nested_bounds_set) {
grand->nested_region_true_start = subord->nested_region_true_start;
grand->nested_region_true_end = subord->nested_region_true_end;
grand->nested_bounds_set = true;
grand->nested_has_no_bounds_ref = false;
pmap_unlock(subord);
} else {
pmap_unlock(subord);
return;
}
pmap_trim_range(grand, grand->nested_region_addr, grand->nested_region_true_start);
pmap_trim_range(grand, grand->nested_region_true_end, (grand->nested_region_addr + grand->nested_region_size));
pmap_trim_subord(subord);
}
MARK_AS_PMAP_TEXT static void
pmap_trim_self(pmap_t pmap)
{
if (pmap->nested_has_no_bounds_ref && pmap->nested_pmap) {
pmap_lock_ro(pmap->nested_pmap);
pmap->nested_has_no_bounds_ref = false;
boolean_t nested_bounds_set = pmap->nested_pmap->nested_bounds_set;
vm_map_offset_t nested_region_true_start = pmap->nested_pmap->nested_region_true_start;
vm_map_offset_t nested_region_true_end = pmap->nested_pmap->nested_region_true_end;
pmap_unlock_ro(pmap->nested_pmap);
if (nested_bounds_set) {
pmap_trim_range(pmap, pmap->nested_region_addr, nested_region_true_start);
pmap_trim_range(pmap, nested_region_true_end, (pmap->nested_region_addr + pmap->nested_region_size));
}
pmap_trim_subord(pmap->nested_pmap);
}
}
MARK_AS_PMAP_TEXT static void
pmap_trim_subord(pmap_t subord)
{
bool contract_subord = false;
pmap_lock(subord);
subord->nested_no_bounds_refcnt--;
if ((subord->nested_no_bounds_refcnt == 0) && (subord->nested_bounds_set)) {
contract_subord = true;
}
pmap_unlock(subord);
if (contract_subord) {
pmap_trim_range(subord, subord->nested_region_addr, subord->nested_region_true_start);
pmap_trim_range(subord, subord->nested_region_true_end, subord->nested_region_addr + subord->nested_region_size);
}
}
void
pmap_trim(
pmap_t grand,
pmap_t subord,
addr64_t vstart,
uint64_t size)
{
#if XNU_MONITOR
pmap_trim_ppl(grand, subord, vstart, size);
pmap_ledger_check_balance(grand);
pmap_ledger_check_balance(subord);
#else
pmap_trim_internal(grand, subord, vstart, size);
#endif
}
#if HAS_APPLE_PAC
static void *
pmap_sign_user_ptr_internal(void *value, ptrauth_key key, uint64_t discriminator, uint64_t jop_key)
{
void *res = NULL;
uint64_t current_intr_state = pmap_interrupts_disable();
uint64_t saved_jop_state = ml_enable_user_jop_key(jop_key);
switch (key) {
case ptrauth_key_asia:
res = ptrauth_sign_unauthenticated(value, ptrauth_key_asia, discriminator);
break;
case ptrauth_key_asda:
res = ptrauth_sign_unauthenticated(value, ptrauth_key_asda, discriminator);
break;
default:
panic("attempt to sign user pointer without process independent key");
}
ml_disable_user_jop_key(jop_key, saved_jop_state);
pmap_interrupts_restore(current_intr_state);
return res;
}
void *
pmap_sign_user_ptr(void *value, ptrauth_key key, uint64_t discriminator, uint64_t jop_key)
{
return pmap_sign_user_ptr_internal(value, key, discriminator, jop_key);
}
static void *
pmap_auth_user_ptr_internal(void *value, ptrauth_key key, uint64_t discriminator, uint64_t jop_key)
{
if ((key != ptrauth_key_asia) && (key != ptrauth_key_asda)) {
panic("attempt to auth user pointer without process independent key");
}
void *res = NULL;
uint64_t current_intr_state = pmap_interrupts_disable();
uint64_t saved_jop_state = ml_enable_user_jop_key(jop_key);
res = ml_auth_ptr_unchecked(value, key, discriminator);
ml_disable_user_jop_key(jop_key, saved_jop_state);
pmap_interrupts_restore(current_intr_state);
return res;
}
void *
pmap_auth_user_ptr(void *value, ptrauth_key key, uint64_t discriminator, uint64_t jop_key)
{
return pmap_auth_user_ptr_internal(value, key, discriminator, jop_key);
}
#endif
MARK_AS_PMAP_TEXT static kern_return_t
pmap_nest_internal(
pmap_t grand,
pmap_t subord,
addr64_t vstart,
uint64_t size)
{
kern_return_t kr = KERN_FAILURE;
vm_map_offset_t vaddr;
tt_entry_t *stte_p;
tt_entry_t *gtte_p;
unsigned int i;
unsigned int num_tte;
unsigned int nested_region_asid_bitmap_size;
unsigned int* nested_region_asid_bitmap;
int expand_options = 0;
bool deref_subord = true;
pmap_t __ptrauth_only subord_addr;
addr64_t vend;
if (__improbable(os_add_overflow(vstart, size, &vend))) {
panic("%s: %p grand addr wraps around: 0x%llx + 0x%llx", __func__, grand, vstart, size);
}
VALIDATE_PMAP(grand);
pmap_reference_internal(subord);
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(grand);
assert(pmap_get_pt_attr(subord) == pt_attr);
#if XNU_MONITOR
expand_options |= PMAP_TT_ALLOCATE_NOWAIT;
#endif
if (__improbable(((size | vstart) & (pt_attr_leaf_table_offmask(pt_attr))) != 0x0ULL)) {
panic("pmap_nest() pmap %p unaligned nesting request 0x%llx, 0x%llx\n", grand, vstart, size);
}
if (__improbable(!subord->nested)) {
panic("%s: subordinate pmap %p is not nestable", __func__, subord);
}
if (subord->nested_region_asid_bitmap == NULL) {
nested_region_asid_bitmap_size = (unsigned int)(size >> pt_attr_twig_shift(pt_attr)) / (sizeof(unsigned int) * NBBY);
#if XNU_MONITOR
pmap_paddr_t pa = 0;
if (__improbable((nested_region_asid_bitmap_size * sizeof(unsigned int)) > PAGE_SIZE)) {
panic("%s: nested_region_asid_bitmap_size=%u will not fit in a page, "
"grand=%p, subord=%p, vstart=0x%llx, size=%llx",
__FUNCTION__, nested_region_asid_bitmap_size,
grand, subord, vstart, size);
}
kr = pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT);
if (kr != KERN_SUCCESS) {
goto nest_cleanup;
}
assert(pa);
nested_region_asid_bitmap = (unsigned int *)phystokv(pa);
#else
nested_region_asid_bitmap = kheap_alloc(KHEAP_DATA_BUFFERS,
nested_region_asid_bitmap_size * sizeof(unsigned int),
Z_WAITOK | Z_ZERO);
#endif
pmap_lock(subord);
if (subord->nested_region_asid_bitmap == NULL) {
subord->nested_region_asid_bitmap_size = nested_region_asid_bitmap_size;
subord->nested_region_addr = vstart;
subord->nested_region_size = (mach_vm_offset_t) size;
__builtin_arm_dmb(DMB_ISHST);
subord->nested_region_asid_bitmap = nested_region_asid_bitmap;
nested_region_asid_bitmap = NULL;
}
pmap_unlock(subord);
if (nested_region_asid_bitmap != NULL) {
#if XNU_MONITOR
pmap_pages_free(kvtophys((vm_offset_t)nested_region_asid_bitmap), PAGE_SIZE);
#else
kheap_free(KHEAP_DATA_BUFFERS, nested_region_asid_bitmap,
nested_region_asid_bitmap_size * sizeof(unsigned int));
#endif
}
}
__builtin_arm_dmb(DMB_ISHLD);
if ((subord->nested_region_addr + subord->nested_region_size) < vend) {
uint64_t new_size;
unsigned int new_nested_region_asid_bitmap_size;
unsigned int* new_nested_region_asid_bitmap;
nested_region_asid_bitmap = NULL;
nested_region_asid_bitmap_size = 0;
new_size = vend - subord->nested_region_addr;
new_nested_region_asid_bitmap_size = (unsigned int)((new_size >> pt_attr_twig_shift(pt_attr)) / (sizeof(unsigned int) * NBBY)) + 1;
#if XNU_MONITOR
pmap_paddr_t pa = 0;
if (__improbable((new_nested_region_asid_bitmap_size * sizeof(unsigned int)) > PAGE_SIZE)) {
panic("%s: new_nested_region_asid_bitmap_size=%u will not fit in a page, "
"grand=%p, subord=%p, vstart=0x%llx, new_size=%llx",
__FUNCTION__, new_nested_region_asid_bitmap_size,
grand, subord, vstart, new_size);
}
kr = pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT);
if (kr != KERN_SUCCESS) {
goto nest_cleanup;
}
assert(pa);
new_nested_region_asid_bitmap = (unsigned int *)phystokv(pa);
#else
new_nested_region_asid_bitmap = kheap_alloc(KHEAP_DATA_BUFFERS,
new_nested_region_asid_bitmap_size * sizeof(unsigned int),
Z_WAITOK | Z_ZERO);
#endif
pmap_lock(subord);
if (subord->nested_region_size < new_size) {
bcopy(subord->nested_region_asid_bitmap,
new_nested_region_asid_bitmap, subord->nested_region_asid_bitmap_size);
nested_region_asid_bitmap_size = subord->nested_region_asid_bitmap_size;
nested_region_asid_bitmap = subord->nested_region_asid_bitmap;
subord->nested_region_asid_bitmap = new_nested_region_asid_bitmap;
subord->nested_region_asid_bitmap_size = new_nested_region_asid_bitmap_size;
subord->nested_region_size = new_size;
new_nested_region_asid_bitmap = NULL;
}
pmap_unlock(subord);
if (nested_region_asid_bitmap != NULL) {
#if XNU_MONITOR
pmap_pages_free(kvtophys((vm_offset_t)nested_region_asid_bitmap), PAGE_SIZE);
#else
kheap_free(KHEAP_DATA_BUFFERS, nested_region_asid_bitmap,
nested_region_asid_bitmap_size * sizeof(unsigned int));
#endif
}
if (new_nested_region_asid_bitmap != NULL) {
#if XNU_MONITOR
pmap_pages_free(kvtophys((vm_offset_t)new_nested_region_asid_bitmap), PAGE_SIZE);
#else
kheap_free(KHEAP_DATA_BUFFERS, new_nested_region_asid_bitmap,
new_nested_region_asid_bitmap_size * sizeof(unsigned int));
#endif
}
}
pmap_lock(subord);
#if __has_feature(ptrauth_calls)
subord_addr = ptrauth_sign_unauthenticated(subord,
ptrauth_key_process_independent_data,
ptrauth_blend_discriminator(&grand->nested_pmap, ptrauth_string_discriminator("pmap.nested_pmap")));
#else
subord_addr = subord;
#endif // __has_feature(ptrauth_calls)
if (os_atomic_cmpxchg(&grand->nested_pmap, PMAP_NULL, subord_addr, relaxed)) {
deref_subord = false;
if (!subord->nested_bounds_set) {
grand->nested_has_no_bounds_ref = true;
subord->nested_no_bounds_refcnt++;
}
grand->nested_region_addr = vstart;
grand->nested_region_size = (mach_vm_offset_t) size;
} else {
if (__improbable(grand->nested_pmap != subord)) {
panic("pmap_nest() pmap %p has a nested pmap\n", grand);
} else if (__improbable(grand->nested_region_addr > vstart)) {
panic("pmap_nest() pmap %p : attempt to nest outside the nested region\n", grand);
} else if ((grand->nested_region_addr + grand->nested_region_size) < vend) {
grand->nested_region_size = (mach_vm_offset_t)(vstart - grand->nested_region_addr + size);
}
}
#if (__ARM_VMSA__ == 7)
vaddr = (vm_map_offset_t) vstart;
num_tte = size >> ARM_TT_L1_SHIFT;
for (i = 0; i < num_tte; i++) {
if (((subord->nested_region_true_start) > vaddr) || ((subord->nested_region_true_end) <= vaddr)) {
goto expand_next;
}
stte_p = pmap_tte(subord, vaddr);
if ((stte_p == (tt_entry_t *)NULL) || (((*stte_p) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)) {
pmap_unlock(subord);
kr = pmap_expand(subord, vaddr, expand_options, PMAP_TT_L2_LEVEL);
if (kr != KERN_SUCCESS) {
pmap_lock(grand);
goto done;
}
pmap_lock(subord);
}
pmap_unlock(subord);
pmap_lock(grand);
stte_p = pmap_tte(grand, vaddr);
if (stte_p == (tt_entry_t *)NULL) {
pmap_unlock(grand);
kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L1_LEVEL);
if (kr != KERN_SUCCESS) {
pmap_lock(grand);
goto done;
}
} else {
pmap_unlock(grand);
kr = KERN_SUCCESS;
}
pmap_lock(subord);
expand_next:
vaddr += ARM_TT_L1_SIZE;
}
#else
vaddr = (vm_map_offset_t) vstart;
num_tte = (unsigned int)(size >> pt_attr_twig_shift(pt_attr));
for (i = 0; i < num_tte; i++) {
if (((subord->nested_region_true_start) > vaddr) || ((subord->nested_region_true_end) <= vaddr)) {
goto expand_next;
}
stte_p = pmap_tte(subord, vaddr);
if (stte_p == PT_ENTRY_NULL || *stte_p == ARM_TTE_EMPTY) {
pmap_unlock(subord);
kr = pmap_expand(subord, vaddr, expand_options, pt_attr_leaf_level(pt_attr));
if (kr != KERN_SUCCESS) {
pmap_lock(grand);
goto done;
}
pmap_lock(subord);
}
expand_next:
vaddr += pt_attr_twig_size(pt_attr);
}
#endif
pmap_unlock(subord);
pmap_lock(grand);
vaddr = (vm_map_offset_t) vstart;
#if (__ARM_VMSA__ == 7)
for (i = 0; i < num_tte; i++) {
if (((subord->nested_region_true_start) > vaddr) || ((subord->nested_region_true_end) <= vaddr)) {
goto nest_next;
}
stte_p = pmap_tte(subord, vaddr);
gtte_p = pmap_tte(grand, vaddr);
*gtte_p = *stte_p;
nest_next:
vaddr += ARM_TT_L1_SIZE;
}
#else
for (i = 0; i < num_tte; i++) {
if (((subord->nested_region_true_start) > vaddr) || ((subord->nested_region_true_end) <= vaddr)) {
goto nest_next;
}
stte_p = pmap_tte(subord, vaddr);
gtte_p = pmap_tte(grand, vaddr);
if (gtte_p == PT_ENTRY_NULL) {
pmap_unlock(grand);
kr = pmap_expand(grand, vaddr, expand_options, pt_attr_twig_level(pt_attr));
pmap_lock(grand);
if (kr != KERN_SUCCESS) {
goto done;
}
gtte_p = pmap_tt2e(grand, vaddr);
}
*gtte_p = *stte_p;
nest_next:
vaddr += pt_attr_twig_size(pt_attr);
}
#endif
kr = KERN_SUCCESS;
done:
stte_p = pmap_tte(grand, vstart);
FLUSH_PTE_RANGE_STRONG(stte_p, stte_p + num_tte);
PMAP_UPDATE_TLBS(grand, vstart, vend, false);
pmap_unlock(grand);
#if XNU_MONITOR
nest_cleanup:
#endif
if (deref_subord) {
pmap_destroy_internal(subord);
}
return kr;
}
kern_return_t
pmap_nest(
pmap_t grand,
pmap_t subord,
addr64_t vstart,
uint64_t size)
{
kern_return_t kr = KERN_FAILURE;
PMAP_TRACE(2, PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord),
VM_KERNEL_ADDRHIDE(vstart));
#if XNU_MONITOR
while ((kr = pmap_nest_ppl(grand, subord, vstart, size)) == KERN_RESOURCE_SHORTAGE) {
pmap_alloc_page_for_ppl(0);
}
pmap_ledger_check_balance(grand);
pmap_ledger_check_balance(subord);
#else
kr = pmap_nest_internal(grand, subord, vstart, size);
#endif
PMAP_TRACE(2, PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, kr);
return kr;
}
kern_return_t
pmap_unnest(
pmap_t grand,
addr64_t vaddr,
uint64_t size)
{
return pmap_unnest_options(grand, vaddr, size, 0);
}
MARK_AS_PMAP_TEXT static kern_return_t
pmap_unnest_options_internal(
pmap_t grand,
addr64_t vaddr,
uint64_t size,
unsigned int option)
{
vm_map_offset_t start;
vm_map_offset_t addr;
tt_entry_t *tte_p;
unsigned int current_index;
unsigned int start_index;
unsigned int max_index;
unsigned int num_tte;
unsigned int i;
addr64_t vend;
if (__improbable(os_add_overflow(vaddr, size, &vend))) {
panic("%s: %p vaddr wraps around: 0x%llx + 0x%llx", __func__, grand, vaddr, size);
}
VALIDATE_PMAP(grand);
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(grand);
if (((size | vaddr) & pt_attr_twig_offmask(pt_attr)) != 0x0ULL) {
panic("pmap_unnest(): unaligned request");
}
if ((option & PMAP_UNNEST_CLEAN) == 0) {
if (grand->nested_pmap == NULL) {
panic("%s: %p has no nested pmap", __func__, grand);
}
if ((vaddr < grand->nested_region_addr) || (vend > (grand->nested_region_addr + grand->nested_region_size))) {
panic("%s: %p: unnest request to region not-fully-nested region [%p, %p)", __func__, grand, (void*)vaddr, (void*)vend);
}
pmap_lock(grand->nested_pmap);
start = vaddr;
start_index = (unsigned int)((vaddr - grand->nested_region_addr) >> pt_attr_twig_shift(pt_attr));
max_index = (unsigned int)(start_index + (size >> pt_attr_twig_shift(pt_attr)));
num_tte = (unsigned int)(size >> pt_attr_twig_shift(pt_attr));
for (current_index = start_index, addr = start; current_index < max_index; current_index++, addr += pt_attr_twig_size(pt_attr)) {
pt_entry_t *bpte, *epte, *cpte;
if (addr < grand->nested_pmap->nested_region_true_start) {
continue;
}
if (addr >= grand->nested_pmap->nested_region_true_end) {
break;
}
bpte = pmap_pte(grand->nested_pmap, addr);
epte = bpte + (pt_attr_leaf_index_mask(pt_attr) >> pt_attr_leaf_shift(pt_attr));
if (!testbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap)) {
setbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap);
for (cpte = bpte; cpte <= epte; cpte++) {
pmap_paddr_t pa;
int pai = 0;
boolean_t managed = FALSE;
pt_entry_t spte;
if ((*cpte != ARM_PTE_TYPE_FAULT)
&& (!ARM_PTE_IS_COMPRESSED(*cpte, cpte))) {
spte = *((volatile pt_entry_t*)cpte);
while (!managed) {
pa = pte_to_pa(spte);
if (!pa_valid(pa)) {
break;
}
pai = (int)pa_index(pa);
LOCK_PVH(pai);
spte = *((volatile pt_entry_t*)cpte);
pa = pte_to_pa(spte);
if (pai == (int)pa_index(pa)) {
managed = TRUE;
break; }
UNLOCK_PVH(pai);
}
if (((spte & ARM_PTE_NG) != ARM_PTE_NG)) {
WRITE_PTE_FAST(cpte, (spte | ARM_PTE_NG));
}
if (managed) {
ASSERT_PVH_LOCKED(pai);
UNLOCK_PVH(pai);
}
}
}
}
FLUSH_PTE_RANGE_STRONG(bpte, epte);
}
flush_mmu_tlb_region_asid_async(vaddr, (unsigned)size, grand->nested_pmap);
sync_tlb_flush();
pmap_unlock(grand->nested_pmap);
}
pmap_lock(grand);
start = vaddr;
addr = vaddr;
num_tte = (unsigned int)(size >> pt_attr_twig_shift(pt_attr));
for (i = 0; i < num_tte; i++, addr += pt_attr_twig_size(pt_attr)) {
if (addr < grand->nested_pmap->nested_region_true_start) {
continue;
}
if (addr >= grand->nested_pmap->nested_region_true_end) {
break;
}
tte_p = pmap_tte(grand, addr);
*tte_p = ARM_TTE_TYPE_FAULT;
}
tte_p = pmap_tte(grand, start);
FLUSH_PTE_RANGE_STRONG(tte_p, tte_p + num_tte);
PMAP_UPDATE_TLBS(grand, start, vend, false);
pmap_unlock(grand);
return KERN_SUCCESS;
}
kern_return_t
pmap_unnest_options(
pmap_t grand,
addr64_t vaddr,
uint64_t size,
unsigned int option)
{
kern_return_t kr = KERN_FAILURE;
PMAP_TRACE(2, PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr));
#if XNU_MONITOR
kr = pmap_unnest_options_ppl(grand, vaddr, size, option);
#else
kr = pmap_unnest_options_internal(grand, vaddr, size, option);
#endif
PMAP_TRACE(2, PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, kr);
return kr;
}
boolean_t
pmap_adjust_unnest_parameters(
__unused pmap_t p,
__unused vm_map_offset_t *s,
__unused vm_map_offset_t *e)
{
return TRUE;
}
#if DEVELOPMENT || DEBUG
void
pmap_disable_NX(
pmap_t pmap)
{
pmap->nx_enabled = FALSE;
}
#else
void
pmap_disable_NX(
__unused pmap_t pmap)
{
}
#endif
#define ARM_FULL_TLB_FLUSH_THRESHOLD 64
#if __ARM_RANGE_TLBI__
#define ARM64_RANGE_TLB_FLUSH_THRESHOLD 1
#define ARM64_FULL_TLB_FLUSH_THRESHOLD ARM64_TLB_RANGE_PAGES
#else
#define ARM64_FULL_TLB_FLUSH_THRESHOLD 256
#endif // __ARM_RANGE_TLBI__
static void
flush_mmu_tlb_region_asid_async(
vm_offset_t va,
size_t length,
pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
vm_offset_t end = va + length;
uint32_t asid;
asid = pmap->hw_asid;
if (length / ARM_SMALL_PAGE_SIZE > ARM_FULL_TLB_FLUSH_THRESHOLD) {
boolean_t flush_all = FALSE;
if ((asid == 0) || (pmap->nested == TRUE)) {
flush_all = TRUE;
}
if (flush_all) {
flush_mmu_tlb_async();
} else {
flush_mmu_tlb_asid_async(asid);
}
return;
}
if (pmap->nested == TRUE) {
#if !__ARM_MP_EXT__
flush_mmu_tlb();
#else
va = arm_trunc_page(va);
while (va < end) {
flush_mmu_tlb_mva_entries_async(va);
va += ARM_SMALL_PAGE_SIZE;
}
#endif
return;
}
va = arm_trunc_page(va) | (asid & 0xff);
flush_mmu_tlb_entries_async(va, end);
#else
unsigned long pmap_page_shift = pt_attr_leaf_shift(pmap_get_pt_attr(pmap));
const uint64_t pmap_page_size = 1ULL << pmap_page_shift;
ppnum_t npages = (ppnum_t)(length >> pmap_page_shift);
uint32_t asid;
asid = pmap->hw_asid;
if (npages > ARM64_FULL_TLB_FLUSH_THRESHOLD) {
boolean_t flush_all = FALSE;
if ((asid == 0) || (pmap->nested == TRUE)) {
flush_all = TRUE;
}
if (flush_all) {
flush_mmu_tlb_async();
} else {
flush_mmu_tlb_asid_async((uint64_t)asid << TLBI_ASID_SHIFT);
}
return;
}
#if __ARM_RANGE_TLBI__
if (npages > ARM64_RANGE_TLB_FLUSH_THRESHOLD) {
va = generate_rtlbi_param(npages, asid, va, pmap_page_shift);
if (pmap->nested == TRUE) {
flush_mmu_tlb_allrange_async(va);
} else {
flush_mmu_tlb_range_async(va);
}
return;
}
#endif
vm_offset_t end = tlbi_asid(asid) | tlbi_addr(va + length);
va = tlbi_asid(asid) | tlbi_addr(va);
if (pmap->nested == TRUE) {
flush_mmu_tlb_allentries_async(va, end, pmap_page_size);
} else {
flush_mmu_tlb_entries_async(va, end, pmap_page_size);
}
#endif
}
MARK_AS_PMAP_TEXT static void
flush_mmu_tlb_full_asid_async(pmap_t pmap)
{
#if (__ARM_VMSA__ == 7)
flush_mmu_tlb_asid_async(pmap->hw_asid);
#else
flush_mmu_tlb_asid_async((uint64_t)(pmap->hw_asid) << TLBI_ASID_SHIFT);
#endif
}
void
flush_mmu_tlb_region(
vm_offset_t va,
unsigned length)
{
flush_mmu_tlb_region_asid_async(va, length, kernel_pmap);
sync_tlb_flush();
}
static pmap_io_range_t*
pmap_find_io_attr(pmap_paddr_t paddr)
{
pmap_io_range_t find_range = {.addr = paddr & ~PAGE_MASK, .len = PAGE_SIZE};
unsigned int begin = 0, end = num_io_rgns - 1;
if ((num_io_rgns == 0) || (paddr < io_attr_table[begin].addr) ||
(paddr >= (io_attr_table[end].addr + io_attr_table[end].len))) {
return NULL;
}
for (;;) {
unsigned int middle = (begin + end) / 2;
int cmp = cmp_io_rgns(&find_range, &io_attr_table[middle]);
if (cmp == 0) {
return &io_attr_table[middle];
} else if (begin == end) {
break;
} else if (cmp > 0) {
begin = middle + 1;
} else {
end = middle;
}
}
return NULL;
}
unsigned int
pmap_cache_attributes(
ppnum_t pn)
{
pmap_paddr_t paddr;
int pai;
unsigned int result;
pp_attr_t pp_attr_current;
paddr = ptoa(pn);
assert(vm_last_phys > vm_first_phys);
if (!pa_valid(paddr)) {
pmap_io_range_t *io_rgn = pmap_find_io_attr(paddr);
return (io_rgn == NULL) ? VM_WIMG_IO : io_rgn->wimg;
}
result = VM_WIMG_DEFAULT;
pai = (int)pa_index(paddr);
pp_attr_current = pp_attr_table[pai];
if (pp_attr_current & PP_ATTR_WIMG_MASK) {
result = pp_attr_current & PP_ATTR_WIMG_MASK;
}
return result;
}
MARK_AS_PMAP_TEXT static void
pmap_sync_wimg(ppnum_t pn, unsigned int wimg_bits_prev, unsigned int wimg_bits_new)
{
if ((wimg_bits_prev != wimg_bits_new)
&& ((wimg_bits_prev == VM_WIMG_COPYBACK)
|| ((wimg_bits_prev == VM_WIMG_INNERWBACK)
&& (wimg_bits_new != VM_WIMG_COPYBACK))
|| ((wimg_bits_prev == VM_WIMG_WTHRU)
&& ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK))))) {
pmap_sync_page_attributes_phys(pn);
}
if ((wimg_bits_new == VM_WIMG_RT) && (wimg_bits_prev != VM_WIMG_RT)) {
pmap_force_dcache_clean(phystokv(ptoa(pn)), PAGE_SIZE);
}
}
MARK_AS_PMAP_TEXT static __unused void
pmap_update_compressor_page_internal(ppnum_t pn, unsigned int prev_cacheattr, unsigned int new_cacheattr)
{
pmap_paddr_t paddr = ptoa(pn);
int pai = (int)pa_index(paddr);
if (__improbable(!pa_valid(paddr))) {
panic("%s called on non-managed page 0x%08x", __func__, pn);
}
LOCK_PVH(pai);
#if XNU_MONITOR
if (__improbable(pa_test_monitor(paddr))) {
panic("%s invoked on PPL page 0x%08x", __func__, pn);
}
#endif
pmap_update_cache_attributes_locked(pn, new_cacheattr);
UNLOCK_PVH(pai);
pmap_sync_wimg(pn, prev_cacheattr & VM_WIMG_MASK, new_cacheattr & VM_WIMG_MASK);
}
void *
pmap_map_compressor_page(ppnum_t pn)
{
#if __ARM_PTE_PHYSMAP__
unsigned int cacheattr = pmap_cache_attributes(pn) & VM_WIMG_MASK;
if (cacheattr != VM_WIMG_DEFAULT) {
#if XNU_MONITOR
pmap_update_compressor_page_ppl(pn, cacheattr, VM_WIMG_DEFAULT);
#else
pmap_update_compressor_page_internal(pn, cacheattr, VM_WIMG_DEFAULT);
#endif
}
#endif
return (void*)phystokv(ptoa(pn));
}
void
pmap_unmap_compressor_page(ppnum_t pn __unused, void *kva __unused)
{
#if __ARM_PTE_PHYSMAP__
unsigned int cacheattr = pmap_cache_attributes(pn) & VM_WIMG_MASK;
if (cacheattr != VM_WIMG_DEFAULT) {
#if XNU_MONITOR
pmap_update_compressor_page_ppl(pn, VM_WIMG_DEFAULT, cacheattr);
#else
pmap_update_compressor_page_internal(pn, VM_WIMG_DEFAULT, cacheattr);
#endif
}
#endif
}
MARK_AS_PMAP_TEXT static boolean_t
pmap_batch_set_cache_attributes_internal(
ppnum_t pn,
unsigned int cacheattr,
unsigned int page_cnt,
unsigned int page_index,
boolean_t doit,
unsigned int *res)
{
pmap_paddr_t paddr;
int pai;
pp_attr_t pp_attr_current;
pp_attr_t pp_attr_template;
unsigned int wimg_bits_prev, wimg_bits_new;
if (cacheattr & VM_WIMG_USE_DEFAULT) {
cacheattr = VM_WIMG_DEFAULT;
}
if ((doit == FALSE) && (*res == 0)) {
pmap_pin_kernel_pages((vm_offset_t)res, sizeof(*res));
*res = page_cnt;
pmap_unpin_kernel_pages((vm_offset_t)res, sizeof(*res));
if (platform_cache_batch_wimg(cacheattr & (VM_WIMG_MASK), page_cnt << PAGE_SHIFT) == FALSE) {
return FALSE;
}
}
paddr = ptoa(pn);
if (!pa_valid(paddr)) {
panic("pmap_batch_set_cache_attributes(): pn 0x%08x not managed", pn);
}
pai = (int)pa_index(paddr);
if (doit) {
LOCK_PVH(pai);
#if XNU_MONITOR
if (pa_test_monitor(paddr)) {
panic("%s invoked on PPL page 0x%llx", __func__, (uint64_t)paddr);
}
#endif
}
do {
pp_attr_current = pp_attr_table[pai];
wimg_bits_prev = VM_WIMG_DEFAULT;
if (pp_attr_current & PP_ATTR_WIMG_MASK) {
wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK;
}
pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK));
if (!doit) {
break;
}
} while (!OSCompareAndSwap16(pp_attr_current, pp_attr_template, &pp_attr_table[pai]));
wimg_bits_new = VM_WIMG_DEFAULT;
if (pp_attr_template & PP_ATTR_WIMG_MASK) {
wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK;
}
if (doit) {
if (wimg_bits_new != wimg_bits_prev) {
pmap_update_cache_attributes_locked(pn, cacheattr);
}
UNLOCK_PVH(pai);
if ((wimg_bits_new == VM_WIMG_RT) && (wimg_bits_prev != VM_WIMG_RT)) {
pmap_force_dcache_clean(phystokv(paddr), PAGE_SIZE);
}
} else {
if (wimg_bits_new == VM_WIMG_COPYBACK) {
return FALSE;
}
if (wimg_bits_prev == wimg_bits_new) {
pmap_pin_kernel_pages((vm_offset_t)res, sizeof(*res));
*res = *res - 1;
pmap_unpin_kernel_pages((vm_offset_t)res, sizeof(*res));
if (!platform_cache_batch_wimg(wimg_bits_new, (*res) << PAGE_SHIFT)) {
return FALSE;
}
}
return TRUE;
}
if (page_cnt == (page_index + 1)) {
wimg_bits_prev = VM_WIMG_COPYBACK;
if (((wimg_bits_prev != wimg_bits_new))
&& ((wimg_bits_prev == VM_WIMG_COPYBACK)
|| ((wimg_bits_prev == VM_WIMG_INNERWBACK)
&& (wimg_bits_new != VM_WIMG_COPYBACK))
|| ((wimg_bits_prev == VM_WIMG_WTHRU)
&& ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK))))) {
platform_cache_flush_wimg(wimg_bits_new);
}
}
return TRUE;
}
boolean_t
pmap_batch_set_cache_attributes(
ppnum_t pn,
unsigned int cacheattr,
unsigned int page_cnt,
unsigned int page_index,
boolean_t doit,
unsigned int *res)
{
#if XNU_MONITOR
return pmap_batch_set_cache_attributes_ppl(pn, cacheattr, page_cnt, page_index, doit, res);
#else
return pmap_batch_set_cache_attributes_internal(pn, cacheattr, page_cnt, page_index, doit, res);
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_set_cache_attributes_priv(
ppnum_t pn,
unsigned int cacheattr,
boolean_t external __unused)
{
pmap_paddr_t paddr;
int pai;
pp_attr_t pp_attr_current;
pp_attr_t pp_attr_template;
unsigned int wimg_bits_prev, wimg_bits_new;
paddr = ptoa(pn);
if (!pa_valid(paddr)) {
return;
}
if (cacheattr & VM_WIMG_USE_DEFAULT) {
cacheattr = VM_WIMG_DEFAULT;
}
pai = (int)pa_index(paddr);
LOCK_PVH(pai);
#if XNU_MONITOR
if (external && pa_test_monitor(paddr)) {
panic("%s invoked on PPL page 0x%llx", __func__, (uint64_t)paddr);
} else if (!external && !pa_test_monitor(paddr)) {
panic("%s invoked on non-PPL page 0x%llx", __func__, (uint64_t)paddr);
}
#endif
do {
pp_attr_current = pp_attr_table[pai];
wimg_bits_prev = VM_WIMG_DEFAULT;
if (pp_attr_current & PP_ATTR_WIMG_MASK) {
wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK;
}
pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK));
} while (!OSCompareAndSwap16(pp_attr_current, pp_attr_template, &pp_attr_table[pai]));
wimg_bits_new = VM_WIMG_DEFAULT;
if (pp_attr_template & PP_ATTR_WIMG_MASK) {
wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK;
}
if (wimg_bits_new != wimg_bits_prev) {
pmap_update_cache_attributes_locked(pn, cacheattr);
}
UNLOCK_PVH(pai);
pmap_sync_wimg(pn, wimg_bits_prev, wimg_bits_new);
}
MARK_AS_PMAP_TEXT static void
pmap_set_cache_attributes_internal(
ppnum_t pn,
unsigned int cacheattr)
{
pmap_set_cache_attributes_priv(pn, cacheattr, TRUE);
}
void
pmap_set_cache_attributes(
ppnum_t pn,
unsigned int cacheattr)
{
#if XNU_MONITOR
pmap_set_cache_attributes_ppl(pn, cacheattr);
#else
pmap_set_cache_attributes_internal(pn, cacheattr);
#endif
}
MARK_AS_PMAP_TEXT void
pmap_update_cache_attributes_locked(
ppnum_t ppnum,
unsigned attributes)
{
pmap_paddr_t phys = ptoa(ppnum);
pv_entry_t *pve_p;
pt_entry_t *pte_p;
pv_entry_t **pv_h;
pt_entry_t tmplate;
unsigned int pai;
boolean_t tlb_flush_needed = FALSE;
PMAP_TRACE(2, PMAP_CODE(PMAP__UPDATE_CACHING) | DBG_FUNC_START, ppnum, attributes);
if (pmap_panic_dev_wimg_on_managed) {
switch (attributes & VM_WIMG_MASK) {
case VM_WIMG_IO: case VM_WIMG_POSTED:
case VM_WIMG_POSTED_REORDERED: case VM_WIMG_POSTED_COMBINED_REORDERED:
panic("%s: trying to use unsupported VM_WIMG type for managed page, VM_WIMG=%x, ppnum=%#x",
__FUNCTION__, attributes & VM_WIMG_MASK, ppnum);
break;
default:
break;
}
}
#if __ARM_PTE_PHYSMAP__
vm_offset_t kva = phystokv(phys);
pte_p = pmap_pte(kernel_pmap, kva);
tmplate = *pte_p;
tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);
#if XNU_MONITOR
tmplate |= (wimg_to_pte(attributes) & ~ARM_PTE_XPRR_MASK);
#else
tmplate |= wimg_to_pte(attributes);
#endif
#if (__ARM_VMSA__ > 7)
if (tmplate & ARM_PTE_HINT_MASK) {
panic("%s: physical aperture PTE %p has hint bit set, va=%p, pte=0x%llx",
__FUNCTION__, pte_p, (void *)kva, tmplate);
}
#endif
WRITE_PTE_STRONG(pte_p, tmplate);
flush_mmu_tlb_region_asid_async(kva, PAGE_SIZE, kernel_pmap);
tlb_flush_needed = TRUE;
#endif
pai = (unsigned int)pa_index(phys);
pv_h = pai_to_pvh(pai);
pte_p = PT_ENTRY_NULL;
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) {
pte_p = pvh_ptep(pv_h);
} else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
pte_p = PT_ENTRY_NULL;
}
while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) {
vm_map_address_t va;
pmap_t pmap;
if (pve_p != PV_ENTRY_NULL) {
pte_p = pve_get_ptep(pve_p);
}
#ifdef PVH_FLAG_IOMMU
if ((vm_offset_t)pte_p & PVH_FLAG_IOMMU) {
goto cache_skip_pve;
}
#endif
pmap = ptep_get_pmap(pte_p);
va = ptep_get_va(pte_p);
tmplate = *pte_p;
tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);
tmplate |= pmap_get_pt_ops(pmap)->wimg_to_pte(attributes);
WRITE_PTE_STRONG(pte_p, tmplate);
pmap_get_pt_ops(pmap)->flush_tlb_region_async(va,
pt_attr_page_size(pmap_get_pt_attr(pmap)) * PAGE_RATIO, pmap);
tlb_flush_needed = TRUE;
#ifdef PVH_FLAG_IOMMU
cache_skip_pve:
#endif
pte_p = PT_ENTRY_NULL;
if (pve_p != PV_ENTRY_NULL) {
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
if (tlb_flush_needed) {
pmap_sync_tlb((attributes & VM_WIMG_MASK) == VM_WIMG_RT);
}
PMAP_TRACE(2, PMAP_CODE(PMAP__UPDATE_CACHING) | DBG_FUNC_END, ppnum, attributes);
}
#if (__ARM_VMSA__ == 7)
void
pmap_create_sharedpages(vm_map_address_t *kernel_data_addr, vm_map_address_t *kernel_text_addr,
vm_map_address_t *user_commpage_addr)
{
pmap_paddr_t pa;
kern_return_t kr;
assert(kernel_data_addr != NULL);
assert(kernel_text_addr != NULL);
assert(user_commpage_addr != NULL);
(void) pmap_pages_alloc_zeroed(&pa, PAGE_SIZE, 0);
kr = pmap_enter(kernel_pmap, _COMM_PAGE_BASE_ADDRESS, atop(pa), VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
*kernel_data_addr = phystokv(pa);
*kernel_text_addr = 0;
*user_commpage_addr = 0;
}
#else
static void
pmap_update_tt3e(
pmap_t pmap,
vm_address_t address,
tt_entry_t template)
{
tt_entry_t *ptep, pte;
ptep = pmap_tt3e(pmap, address);
if (ptep == NULL) {
panic("%s: no ptep?\n", __FUNCTION__);
}
pte = *ptep;
pte = tte_to_pa(pte) | template;
WRITE_PTE_STRONG(ptep, pte);
}
#define PMAP_COMM_PAGE_PTE_TEMPLATE (ARM_PTE_TYPE_VALID \
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \
| ARM_PTE_SH(SH_INNER_MEMORY) | ARM_PTE_NX \
| ARM_PTE_PNX | ARM_PTE_AP(AP_RORO) | ARM_PTE_AF)
#define PMAP_COMM_PAGE_TEXT_PTE_TEMPLATE (ARM_PTE_TYPE_VALID \
| ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \
| ARM_PTE_SH(SH_INNER_MEMORY) | ARM_PTE_PNX \
| ARM_PTE_AP(AP_RORO) | ARM_PTE_AF)
void
pmap_create_sharedpages(vm_map_address_t *kernel_data_addr, vm_map_address_t *kernel_text_addr,
vm_map_address_t *user_text_addr)
{
kern_return_t kr;
pmap_paddr_t data_pa = 0; pmap_paddr_t text_pa = 0;
*kernel_data_addr = 0;
*kernel_text_addr = 0;
*user_text_addr = 0;
#if XNU_MONITOR
data_pa = pmap_alloc_page_for_kern(0);
assert(data_pa);
memset((char *) phystokv(data_pa), 0, PAGE_SIZE);
#if CONFIG_ARM_PFZ
text_pa = pmap_alloc_page_for_kern(0);
assert(text_pa);
memset((char *) phystokv(text_pa), 0, PAGE_SIZE);
#endif
#else
(void) pmap_pages_alloc_zeroed(&data_pa, PAGE_SIZE, 0);
#if CONFIG_ARM_PFZ
(void) pmap_pages_alloc_zeroed(&text_pa, PAGE_SIZE, 0);
#endif
#endif
#ifdef CONFIG_XNUPOST
kr = pmap_enter(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, (ppnum_t)atop(data_pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE | ARM_PTE_NG);
#if KASAN
kasan_map_shadow(_COMM_HIGH_PAGE64_BASE_ADDRESS, PAGE_SIZE, true);
#endif
#endif
sharedpage_pmap_default = pmap_create_options(NULL, 0x0, 0);
assert(sharedpage_pmap_default != NULL);
kr = pmap_enter_addr(sharedpage_pmap_default, _COMM_PAGE64_BASE_ADDRESS, data_pa, VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap_default, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
#if CONFIG_ARM_PFZ
const pt_attr_t * const pt_attr = pmap_get_pt_attr(sharedpage_pmap_default);
int num_ptes = pt_attr_leaf_size(pt_attr) >> PTE_SHIFT;
vm_map_address_t commpage_text_va = 0;
do {
int text_leaf_index = random() % num_ptes;
commpage_text_va = (_COMM_PAGE64_BASE_ADDRESS & ~pt_attr_leaf_index_mask(pt_attr));
commpage_text_va |= (text_leaf_index << pt_attr_leaf_shift(pt_attr));
} while (commpage_text_va == _COMM_PAGE64_BASE_ADDRESS);
__assert_only pt_entry_t *ptep = pmap_pte(sharedpage_pmap_default, commpage_text_va);
assert(ptep != PT_ENTRY_NULL);
assert(*ptep == ARM_TTE_EMPTY);
kr = pmap_enter_addr(sharedpage_pmap_default, commpage_text_va, text_pa, VM_PROT_READ | VM_PROT_EXECUTE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap_default, commpage_text_va, PMAP_COMM_PAGE_TEXT_PTE_TEMPLATE);
*user_text_addr = commpage_text_va;
#endif
kr = pmap_enter_addr(sharedpage_pmap_default, _COMM_PAGE32_BASE_ADDRESS, data_pa, VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap_default, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
#if __ARM_MIXED_PAGE_SIZE__
sharedpage_pmap_4k = pmap_create_options(NULL, 0x0, PMAP_CREATE_FORCE_4K_PAGES);
assert(sharedpage_pmap_4k != NULL);
kr = pmap_enter_addr(sharedpage_pmap_4k, _COMM_PAGE64_BASE_ADDRESS, data_pa, VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap_4k, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
kr = pmap_enter_addr(sharedpage_pmap_4k, _COMM_PAGE32_BASE_ADDRESS, data_pa, VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
assert(kr == KERN_SUCCESS);
pmap_update_tt3e(sharedpage_pmap_4k, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
#endif
*kernel_data_addr = phystokv(data_pa);
*kernel_text_addr = (text_pa) ? phystokv(text_pa) : 0;
return;
}
#if (ARM_PGSHIFT == 14)
static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= VM_MAX_ADDRESS);
#elif (ARM_PGSHIFT == 12)
static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= VM_MAX_ADDRESS);
#else
#error Nested shared page mapping is unsupported on this config
#endif
MARK_AS_PMAP_TEXT static kern_return_t
pmap_insert_sharedpage_internal(
pmap_t pmap)
{
kern_return_t kr = KERN_SUCCESS;
vm_offset_t sharedpage_vaddr;
pt_entry_t *ttep, *src_ttep;
int options = 0;
pmap_t sharedpage_pmap = sharedpage_pmap_default;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
const unsigned int sharedpage_level = pt_attr_sharedpage_level(pt_attr);
#if __ARM_MIXED_PAGE_SIZE__
#if !__ARM_16K_PG__
#error "pmap_insert_sharedpage_internal requires a 16KB default kernel page size when __ARM_MIXED_PAGE_SIZE__ is enabled"
#endif
const uint64_t pmap_page_size = pt_attr_page_size(pt_attr);
if (pmap_page_size == 16384) {
sharedpage_pmap = sharedpage_pmap_default;
} else if (pmap_page_size == 4096) {
sharedpage_pmap = sharedpage_pmap_4k;
} else {
panic("No shared page pmap exists for the wanted page size: %llu", pmap_page_size);
}
#endif
VALIDATE_PMAP(pmap);
#if XNU_MONITOR
options |= PMAP_OPTIONS_NOWAIT;
#endif
#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
#error We assume a single page.
#endif
if (pmap_is_64bit(pmap)) {
sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS;
} else {
sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS;
}
pmap_lock(pmap);
while ((ttep = pmap_ttne(pmap, sharedpage_level, sharedpage_vaddr)) == TT_ENTRY_NULL) {
pmap_unlock(pmap);
kr = pmap_expand(pmap, sharedpage_vaddr, options, sharedpage_level);
if (kr != KERN_SUCCESS) {
#if XNU_MONITOR
if (kr == KERN_RESOURCE_SHORTAGE) {
return kr;
} else
#endif
{
panic("Failed to pmap_expand for commpage, pmap=%p", pmap);
}
}
pmap_lock(pmap);
}
if (*ttep != ARM_PTE_EMPTY) {
panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
}
src_ttep = pmap_ttne(sharedpage_pmap, sharedpage_level, sharedpage_vaddr);
*ttep = *src_ttep;
FLUSH_PTE_STRONG(ttep);
pmap_unlock(pmap);
return kr;
}
static void
pmap_unmap_sharedpage(
pmap_t pmap)
{
pt_entry_t *ttep;
vm_offset_t sharedpage_vaddr;
pmap_t sharedpage_pmap = sharedpage_pmap_default;
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
const unsigned int sharedpage_level = pt_attr_sharedpage_level(pt_attr);
#if __ARM_MIXED_PAGE_SIZE__
#if !__ARM_16K_PG__
#error "pmap_unmap_sharedpage requires a 16KB default kernel page size when __ARM_MIXED_PAGE_SIZE__ is enabled"
#endif
const uint64_t pmap_page_size = pt_attr_page_size(pt_attr);
if (pmap_page_size == 16384) {
sharedpage_pmap = sharedpage_pmap_default;
} else if (pmap_page_size == 4096) {
sharedpage_pmap = sharedpage_pmap_4k;
} else {
panic("No shared page pmap exists for the wanted page size: %llu", pmap_page_size);
}
#endif
#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
#error We assume a single page.
#endif
if (pmap_is_64bit(pmap)) {
sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS;
} else {
sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS;
}
ttep = pmap_ttne(pmap, sharedpage_level, sharedpage_vaddr);
if (ttep == NULL) {
return;
}
if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_ttne(sharedpage_pmap, sharedpage_level, sharedpage_vaddr)) {
panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
}
*ttep = ARM_TTE_EMPTY;
FLUSH_PTE_STRONG(ttep);
flush_mmu_tlb_region_asid_async(sharedpage_vaddr, PAGE_SIZE, pmap);
sync_tlb_flush();
}
void
pmap_insert_sharedpage(
pmap_t pmap)
{
#if XNU_MONITOR
kern_return_t kr = KERN_FAILURE;
while ((kr = pmap_insert_sharedpage_ppl(pmap)) == KERN_RESOURCE_SHORTAGE) {
pmap_alloc_page_for_ppl(0);
}
pmap_ledger_check_balance(pmap);
if (kr != KERN_SUCCESS) {
panic("%s: failed to insert the shared page, kr=%d, "
"pmap=%p",
__FUNCTION__, kr,
pmap);
}
#else
pmap_insert_sharedpage_internal(pmap);
#endif
}
static boolean_t
pmap_is_64bit(
pmap_t pmap)
{
return pmap->is_64bit;
}
bool
pmap_is_exotic(
pmap_t pmap __unused)
{
return false;
}
#endif
boolean_t
pmap_valid_page(
ppnum_t pn)
{
return pa_valid(ptoa(pn));
}
boolean_t
pmap_bootloader_page(
ppnum_t pn)
{
pmap_paddr_t paddr = ptoa(pn);
if (pa_valid(paddr)) {
return FALSE;
}
pmap_io_range_t *io_rgn = pmap_find_io_attr(paddr);
return (io_rgn != NULL) && (io_rgn->wimg & PMAP_IO_RANGE_CARVEOUT);
}
MARK_AS_PMAP_TEXT static boolean_t
pmap_is_empty_internal(
pmap_t pmap,
vm_map_offset_t va_start,
vm_map_offset_t va_end)
{
vm_map_offset_t block_start, block_end;
tt_entry_t *tte_p;
if (pmap == NULL) {
return TRUE;
}
VALIDATE_PMAP(pmap);
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
unsigned int initial_not_in_kdp = not_in_kdp;
if ((pmap != kernel_pmap) && (initial_not_in_kdp)) {
pmap_lock_ro(pmap);
}
#if (__ARM_VMSA__ == 7)
if (tte_index(pmap, pt_attr, va_end) >= pmap->tte_index_max) {
if ((pmap != kernel_pmap) && (initial_not_in_kdp)) {
pmap_unlock_ro(pmap);
}
return TRUE;
}
#endif
block_start = va_start;
while (block_start < va_end) {
pt_entry_t *bpte_p, *epte_p;
pt_entry_t *pte_p;
block_end = (block_start + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr);
if (block_end > va_end) {
block_end = va_end;
}
tte_p = pmap_tte(pmap, block_start);
if ((tte_p != PT_ENTRY_NULL)
&& ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte_p = &pte_p[pte_index(pmap, pt_attr, block_start)];
epte_p = &pte_p[pte_index(pmap, pt_attr, block_end)];
for (pte_p = bpte_p; pte_p < epte_p; pte_p++) {
if (*pte_p != ARM_PTE_EMPTY) {
if ((pmap != kernel_pmap) && (initial_not_in_kdp)) {
pmap_unlock_ro(pmap);
}
return FALSE;
}
}
}
block_start = block_end;
}
if ((pmap != kernel_pmap) && (initial_not_in_kdp)) {
pmap_unlock_ro(pmap);
}
return TRUE;
}
boolean_t
pmap_is_empty(
pmap_t pmap,
vm_map_offset_t va_start,
vm_map_offset_t va_end)
{
#if XNU_MONITOR
return pmap_is_empty_ppl(pmap, va_start, va_end);
#else
return pmap_is_empty_internal(pmap, va_start, va_end);
#endif
}
vm_map_offset_t
pmap_max_offset(
boolean_t is64,
unsigned int option)
{
return (is64) ? pmap_max_64bit_offset(option) : pmap_max_32bit_offset(option);
}
vm_map_offset_t
pmap_max_64bit_offset(
__unused unsigned int option)
{
vm_map_offset_t max_offset_ret = 0;
#if defined(__arm64__)
#define ARM64_MIN_MAX_ADDRESS (SHARED_REGION_BASE_ARM64 + SHARED_REGION_SIZE_ARM64 + 0x20000000) _Static_assert((ARM64_MIN_MAX_ADDRESS > SHARED_REGION_BASE_ARM64) && (ARM64_MIN_MAX_ADDRESS <= MACH_VM_MAX_ADDRESS),
"Minimum address space size outside allowable range");
const vm_map_offset_t min_max_offset = ARM64_MIN_MAX_ADDRESS; if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) {
max_offset_ret = arm64_pmap_max_offset_default;
} else if (option == ARM_PMAP_MAX_OFFSET_MIN) {
max_offset_ret = min_max_offset;
} else if (option == ARM_PMAP_MAX_OFFSET_MAX) {
max_offset_ret = MACH_VM_MAX_ADDRESS;
} else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) {
if (arm64_pmap_max_offset_default) {
max_offset_ret = arm64_pmap_max_offset_default;
} else if (max_mem > 0xC0000000) {
max_offset_ret = min_max_offset + 0x138000000; } else if (max_mem > 0x40000000) {
max_offset_ret = min_max_offset + 0x38000000; } else {
max_offset_ret = min_max_offset;
}
} else if (option == ARM_PMAP_MAX_OFFSET_JUMBO) {
if (arm64_pmap_max_offset_default) {
max_offset_ret = arm64_pmap_max_offset_default;
} else {
max_offset_ret = MACH_VM_MAX_ADDRESS; }
} else {
panic("pmap_max_64bit_offset illegal option 0x%x\n", option);
}
assert(max_offset_ret <= MACH_VM_MAX_ADDRESS);
assert(max_offset_ret >= min_max_offset);
#else
panic("Can't run pmap_max_64bit_offset on non-64bit architectures\n");
#endif
return max_offset_ret;
}
vm_map_offset_t
pmap_max_32bit_offset(
unsigned int option)
{
vm_map_offset_t max_offset_ret = 0;
if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) {
max_offset_ret = arm_pmap_max_offset_default;
} else if (option == ARM_PMAP_MAX_OFFSET_MIN) {
max_offset_ret = 0x80000000;
} else if (option == ARM_PMAP_MAX_OFFSET_MAX) {
max_offset_ret = VM_MAX_ADDRESS;
} else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) {
if (arm_pmap_max_offset_default) {
max_offset_ret = arm_pmap_max_offset_default;
} else if (max_mem > 0x20000000) {
max_offset_ret = 0x80000000;
} else {
max_offset_ret = 0x80000000;
}
} else if (option == ARM_PMAP_MAX_OFFSET_JUMBO) {
max_offset_ret = 0x80000000;
} else {
panic("pmap_max_32bit_offset illegal option 0x%x\n", option);
}
assert(max_offset_ret <= MACH_VM_MAX_ADDRESS);
return max_offset_ret;
}
#if CONFIG_DTRACE
extern kern_return_t dtrace_copyio_preflight(addr64_t);
extern kern_return_t dtrace_copyio_postflight(addr64_t);
kern_return_t
dtrace_copyio_preflight(
__unused addr64_t va)
{
if (current_map() == kernel_map) {
return KERN_FAILURE;
} else {
return KERN_SUCCESS;
}
}
kern_return_t
dtrace_copyio_postflight(
__unused addr64_t va)
{
return KERN_SUCCESS;
}
#endif
void
pmap_flush_context_init(__unused pmap_flush_context *pfc)
{
}
void
pmap_flush(
__unused pmap_flush_context *cpus_to_flush)
{
return;
}
#if XNU_MONITOR
static void
pmap_pin_kernel_pages(vm_offset_t kva, size_t nbytes)
{
vm_offset_t end;
if (os_add_overflow(kva, nbytes, &end)) {
panic("%s(%p, 0x%llx): overflow", __func__, (void*)kva, (uint64_t)nbytes);
}
for (vm_offset_t ckva = kva; ckva < end; ckva = round_page(ckva + 1)) {
pmap_paddr_t pa = kvtophys(ckva);
if (!pa_valid(pa)) {
panic("%s(%p): invalid physical page 0x%llx", __func__, (void*)kva, (uint64_t)pa);
}
pp_attr_t attr;
unsigned int pai = (unsigned int)pa_index(pa);
if (ckva == phystokv(pa)) {
panic("%s(%p): attempt to pin static mapping for page 0x%llx", __func__, (void*)kva, (uint64_t)pa);
}
do {
attr = pp_attr_table[pai] & ~PP_ATTR_NO_MONITOR;
if (attr & PP_ATTR_MONITOR) {
panic("%s(%p): physical page 0x%llx belongs to PPL", __func__, (void*)kva, (uint64_t)pa);
}
} while (!OSCompareAndSwap16(attr, attr | PP_ATTR_NO_MONITOR, &pp_attr_table[pai]));
}
}
static void
pmap_unpin_kernel_pages(vm_offset_t kva, size_t nbytes)
{
vm_offset_t end;
if (os_add_overflow(kva, nbytes, &end)) {
panic("%s(%p, 0x%llx): overflow", __func__, (void*)kva, (uint64_t)nbytes);
}
for (vm_offset_t ckva = kva; ckva < end; ckva = round_page(ckva + 1)) {
pmap_paddr_t pa = kvtophys(ckva);
if (!pa_valid(pa)) {
panic("%s(%p): invalid physical page 0x%llx", __func__, (void*)kva, (uint64_t)pa);
}
if (!(pp_attr_table[pa_index(pa)] & PP_ATTR_NO_MONITOR)) {
panic("%s(%p): physical page 0x%llx not pinned", __func__, (void*)kva, (uint64_t)pa);
}
assert(!(pp_attr_table[pa_index(pa)] & PP_ATTR_MONITOR));
pa_clear_no_monitor(pa);
}
}
MARK_AS_PMAP_TEXT static void
pmap_ppl_lockdown_page(vm_address_t kva)
{
pmap_paddr_t pa = kvtophys(kva);
unsigned int pai = (unsigned int)pa_index(pa);
LOCK_PVH(pai);
pv_entry_t **pv_h = pai_to_pvh(pai);
if (__improbable(pa_test_monitor(pa))) {
panic("%#lx: page %llx belongs to PPL", kva, pa);
}
if (__improbable(pvh_get_flags(pv_h) & (PVH_FLAG_LOCKDOWN | PVH_FLAG_EXEC))) {
panic("%#lx: already locked down/executable (%#llx)", kva, pvh_get_flags(pv_h));
}
pt_entry_t *pte_p = pmap_pte(kernel_pmap, kva);
if (pte_p == PT_ENTRY_NULL) {
panic("%#lx: NULL pte", kva);
}
pt_entry_t tmplate = *pte_p;
if (__improbable((tmplate & ARM_PTE_APMASK) != ARM_PTE_AP(AP_RWNA))) {
panic("%#lx: not a kernel r/w page (%#llx)", kva, tmplate & ARM_PTE_APMASK);
}
pvh_set_flags(pv_h, pvh_get_flags(pv_h) | PVH_FLAG_LOCKDOWN);
pmap_set_ptov_ap(pai, AP_RONA, FALSE);
UNLOCK_PVH(pai);
pmap_page_protect_options_internal((ppnum_t)atop(pa), VM_PROT_READ, 0, NULL);
}
MARK_AS_PMAP_TEXT static void
pmap_ppl_unlockdown_page(vm_address_t kva)
{
pmap_paddr_t pa = kvtophys(kva);
unsigned int pai = (unsigned int)pa_index(pa);
LOCK_PVH(pai);
pv_entry_t **pv_h = pai_to_pvh(pai);
vm_offset_t pvh_flags = pvh_get_flags(pv_h);
if (__improbable(!(pvh_flags & PVH_FLAG_LOCKDOWN))) {
panic("unlockdown attempt on not locked down virtual %#lx/pai %d", kva, pai);
}
pvh_set_flags(pv_h, pvh_flags & ~PVH_FLAG_LOCKDOWN);
pmap_set_ptov_ap(pai, AP_RWNA, FALSE);
UNLOCK_PVH(pai);
}
#else
static void __unused
pmap_pin_kernel_pages(vm_offset_t kva __unused, size_t nbytes __unused)
{
}
static void __unused
pmap_unpin_kernel_pages(vm_offset_t kva __unused, size_t nbytes __unused)
{
}
#endif
#define PMAP_RESIDENT_INVALID ((mach_vm_size_t)-1)
MARK_AS_PMAP_TEXT static mach_vm_size_t
pmap_query_resident_internal(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
mach_vm_size_t *compressed_bytes_p)
{
mach_vm_size_t resident_bytes = 0;
mach_vm_size_t compressed_bytes = 0;
pt_entry_t *bpte, *epte;
pt_entry_t *pte_p;
tt_entry_t *tte_p;
if (pmap == NULL) {
return PMAP_RESIDENT_INVALID;
}
VALIDATE_PMAP(pmap);
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
if (__improbable((start % pt_attr_page_size(pt_attr)) ||
(end % pt_attr_page_size(pt_attr)))) {
panic("%s: address range %p, %p not page-aligned to 0x%llx", __func__, (void*)start, (void*)end, pt_attr_page_size(pt_attr));
}
if (__improbable((end < start) || (end > ((start + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr))))) {
panic("%s: invalid address range %p, %p", __func__, (void*)start, (void*)end);
}
pmap_lock_ro(pmap);
tte_p = pmap_tte(pmap, start);
if (tte_p == (tt_entry_t *) NULL) {
pmap_unlock_ro(pmap);
return PMAP_RESIDENT_INVALID;
}
if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) {
pte_p = (pt_entry_t *) ttetokv(*tte_p);
bpte = &pte_p[pte_index(pmap, pt_attr, start)];
epte = &pte_p[pte_index(pmap, pt_attr, end)];
for (; bpte < epte; bpte++) {
if (ARM_PTE_IS_COMPRESSED(*bpte, bpte)) {
compressed_bytes += pt_attr_page_size(pt_attr);
} else if (pa_valid(pte_to_pa(*bpte))) {
resident_bytes += pt_attr_page_size(pt_attr);
}
}
}
pmap_unlock_ro(pmap);
if (compressed_bytes_p) {
pmap_pin_kernel_pages((vm_offset_t)compressed_bytes_p, sizeof(*compressed_bytes_p));
*compressed_bytes_p += compressed_bytes;
pmap_unpin_kernel_pages((vm_offset_t)compressed_bytes_p, sizeof(*compressed_bytes_p));
}
return resident_bytes;
}
mach_vm_size_t
pmap_query_resident(
pmap_t pmap,
vm_map_address_t start,
vm_map_address_t end,
mach_vm_size_t *compressed_bytes_p)
{
mach_vm_size_t total_resident_bytes;
mach_vm_size_t compressed_bytes;
vm_map_address_t va;
if (pmap == PMAP_NULL) {
if (compressed_bytes_p) {
*compressed_bytes_p = 0;
}
return 0;
}
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
total_resident_bytes = 0;
compressed_bytes = 0;
PMAP_TRACE(3, PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START,
VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start),
VM_KERNEL_ADDRHIDE(end));
va = start;
while (va < end) {
vm_map_address_t l;
mach_vm_size_t resident_bytes;
l = ((va + pt_attr_twig_size(pt_attr)) & ~pt_attr_twig_offmask(pt_attr));
if (l > end) {
l = end;
}
#if XNU_MONITOR
resident_bytes = pmap_query_resident_ppl(pmap, va, l, compressed_bytes_p);
#else
resident_bytes = pmap_query_resident_internal(pmap, va, l, compressed_bytes_p);
#endif
if (resident_bytes == PMAP_RESIDENT_INVALID) {
break;
}
total_resident_bytes += resident_bytes;
va = l;
}
if (compressed_bytes_p) {
*compressed_bytes_p = compressed_bytes;
}
PMAP_TRACE(3, PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END,
total_resident_bytes);
return total_resident_bytes;
}
#if MACH_ASSERT
static void
pmap_check_ledgers(
pmap_t pmap)
{
int pid;
char *procname;
if (pmap->pmap_pid == 0) {
return;
}
pid = pmap->pmap_pid;
procname = pmap->pmap_procname;
vm_map_pmap_check_ledgers(pmap, pmap->ledger, pid, procname);
PMAP_STATS_ASSERTF(pmap->stats.resident_count == 0, pmap, "stats.resident_count %d", pmap->stats.resident_count);
#if 00
PMAP_STATS_ASSERTF(pmap->stats.wired_count == 0, pmap, "stats.wired_count %d", pmap->stats.wired_count);
#endif
PMAP_STATS_ASSERTF(pmap->stats.device == 0, pmap, "stats.device %d", pmap->stats.device);
PMAP_STATS_ASSERTF(pmap->stats.internal == 0, pmap, "stats.internal %d", pmap->stats.internal);
PMAP_STATS_ASSERTF(pmap->stats.external == 0, pmap, "stats.external %d", pmap->stats.external);
PMAP_STATS_ASSERTF(pmap->stats.reusable == 0, pmap, "stats.reusable %d", pmap->stats.reusable);
PMAP_STATS_ASSERTF(pmap->stats.compressed == 0, pmap, "stats.compressed %lld", pmap->stats.compressed);
}
#endif
void
pmap_advise_pagezero_range(__unused pmap_t p, __unused uint64_t a)
{
}
#if CONFIG_PGTRACE
#define PROF_START uint64_t t, nanot;\
t = mach_absolute_time();
#define PROF_END absolutetime_to_nanoseconds(mach_absolute_time()-t, &nanot);\
kprintf("%s: took %llu ns\n", __func__, nanot);
#define PMAP_PGTRACE_LOCK(p) \
do { \
*(p) = pmap_interrupts_disable(); \
if (simple_lock_try(&(pmap_pgtrace.lock), LCK_GRP_NULL)) break; \
pmap_interrupts_restore(*(p)); \
} while (true)
#define PMAP_PGTRACE_UNLOCK(p) \
do { \
simple_unlock(&(pmap_pgtrace.lock)); \
pmap_interrupts_restore(*(p)); \
} while (0)
#define PGTRACE_WRITE_PTE(pte_p, pte_entry) \
do { \
*(pte_p) = (pte_entry); \
FLUSH_PTE(pte_p); \
} while (0)
#define PGTRACE_MAX_MAP 16 // maximum supported va to same pa
typedef enum {
UNDEFINED,
PA_UNDEFINED,
VA_UNDEFINED,
DEFINED
} pmap_pgtrace_page_state_t;
typedef struct {
queue_chain_t chain;
pmap_paddr_t pa;
queue_head_t maps;
queue_head_t map_pool;
queue_head_t map_waste;
pmap_pgtrace_page_state_t state;
} pmap_pgtrace_page_t;
typedef struct {
queue_chain_t chain;
pmap_t pmap;
vm_map_offset_t va;
} pmap_va_t;
static ZONE_VIEW_DEFINE(ZV_PMAP_VA, "pmap va",
KHEAP_ID_DEFAULT, sizeof(pmap_va_t));
static ZONE_VIEW_DEFINE(ZV_PMAP_PGTRACE, "pmap pgtrace",
KHEAP_ID_DEFAULT, sizeof(pmap_pgtrace_page_t));
static struct {
queue_head_t pages;
decl_simple_lock_data(, lock);
} pmap_pgtrace = {};
static void
pmap_pgtrace_init(void)
{
queue_init(&(pmap_pgtrace.pages));
simple_lock_init(&(pmap_pgtrace.lock), 0);
boolean_t enabled;
if (PE_parse_boot_argn("pgtrace", &enabled, sizeof(enabled))) {
pgtrace_enabled = enabled;
}
}
inline static pmap_pgtrace_page_t *
pmap_pgtrace_find_page(pmap_paddr_t pa)
{
queue_head_t *q = &(pmap_pgtrace.pages);
pmap_pgtrace_page_t *p;
queue_iterate(q, p, pmap_pgtrace_page_t *, chain) {
if (p->state == UNDEFINED) {
continue;
}
if (p->state == PA_UNDEFINED) {
continue;
}
if (p->pa == pa) {
return p;
}
}
return NULL;
}
static bool
pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end)
{
uint64_t ints;
queue_head_t *q = &(pmap_pgtrace.pages);
pmap_paddr_t pa_page;
pt_entry_t *ptep, *cptep;
pmap_pgtrace_page_t *p;
bool found = false;
pmap_assert_locked_w(pmap);
assert(va_page == arm_trunc_page(va_page));
PMAP_PGTRACE_LOCK(&ints);
ptep = pmap_pte(pmap, va_page);
if (!ptep || !(*ptep & ARM_PTE_TYPE_VALID)) {
PMAP_PGTRACE_UNLOCK(&ints);
return false;
}
queue_head_t *mapq;
queue_head_t *mappool;
pmap_pgtrace_map_t *map = NULL;
pa_page = pte_to_pa(*ptep);
queue_iterate(q, p, pmap_pgtrace_page_t *, chain) {
mapq = &(p->maps);
mappool = &(p->map_pool);
switch (p->state) {
case PA_UNDEFINED:
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == false && map->pmap == pmap && map->ova == va_page) {
p->pa = pa_page;
map->range.start = start;
map->range.end = end;
found = true;
break;
}
}
break;
case VA_UNDEFINED:
if (p->pa != pa_page) {
break;
}
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == false) {
map->pmap = pmap;
map->ova = va_page;
map->range.start = start;
map->range.end = end;
found = true;
break;
}
}
break;
case DEFINED:
if (p->pa != pa_page) {
break;
}
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == true && map->pmap == pmap && map->ova == va_page) {
kprintf("%s: skip existing mapping at va=%llx\n", __func__, va_page);
break;
} else if (map->cloned == true && map->pmap == kernel_pmap && map->cva[1] == va_page) {
kprintf("%s: skip clone mapping at va=%llx\n", __func__, va_page);
break;
} else if (map->cloned == false && map->pmap == pmap && map->ova == va_page) {
found = true;
break;
}
}
break;
default:
panic("invalid state p->state=%x\n", p->state);
}
if (found == true) {
break;
}
}
if (found == false) {
PMAP_PGTRACE_UNLOCK(&ints);
return false;
}
for (int i = 0; i < 3; i++) {
ptep = pmap_pte(pmap, va_page + (i - 1) * ARM_PGBYTES);
cptep = pmap_pte(kernel_pmap, map->cva[i]);
assert(cptep != NULL);
if (ptep == NULL) {
PGTRACE_WRITE_PTE(cptep, (pt_entry_t)NULL);
} else {
PGTRACE_WRITE_PTE(cptep, *ptep);
}
PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i] + ARM_PGBYTES, false);
}
ptep = pmap_pte(pmap, va_page);
cptep = pmap_pte(kernel_pmap, map->cva[1]);
PGTRACE_WRITE_PTE(ptep, (*ptep | ARM_PTE_PGTRACE) & ~ARM_PTE_TYPE_VALID);
PMAP_UPDATE_TLBS(pmap, map->ova, map->ova + ARM_PGBYTES, false);
map->cloned = true;
p->state = DEFINED;
kprintf("%s: pa_page=%llx va_page=%llx cva[1]=%llx pmap=%p ptep=%p cptep=%p\n", __func__, pa_page, va_page, map->cva[1], pmap, ptep, cptep);
PMAP_PGTRACE_UNLOCK(&ints);
return true;
}
static void
pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t va)
{
uint64_t ints, found = false;
pmap_pgtrace_page_t *p;
pt_entry_t *ptep;
PMAP_PGTRACE_LOCK(&ints);
p = pmap_pgtrace_find_page(pa);
if (p == NULL) {
goto unlock_exit;
}
queue_head_t *mapq = &(p->maps);
queue_head_t *mappool = &(p->map_pool);
pmap_pgtrace_map_t *map;
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->pmap == pmap && map->ova == va) {
found = true;
break;
}
}
if (!found) {
goto unlock_exit;
}
if (map->cloned == true) {
ptep = pmap_pte(pmap, map->ova);
assert(ptep);
PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID);
PMAP_UPDATE_TLBS(pmap, va, va + ARM_PGBYTES, false);
for (int i = 0; i < 3; i++) {
ptep = pmap_pte(kernel_pmap, map->cva[i]);
assert(ptep != NULL);
PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]);
PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i] + ARM_PGBYTES, false);
}
}
queue_remove(mapq, map, pmap_pgtrace_map_t *, chain);
map->pmap = NULL;
map->ova = (vm_map_offset_t)NULL;
map->cloned = false;
queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain);
kprintf("%s: p=%p pa=%llx va=%llx\n", __func__, p, pa, va);
unlock_exit:
PMAP_PGTRACE_UNLOCK(&ints);
}
static void
pmap_pgtrace_remove_all_clone(pmap_paddr_t pa)
{
uint64_t ints;
pmap_pgtrace_page_t *p;
pt_entry_t *ptep;
PMAP_PGTRACE_LOCK(&ints);
p = pmap_pgtrace_find_page(pa);
if (p == NULL) {
PMAP_PGTRACE_UNLOCK(&ints);
return;
}
queue_head_t *mapq = &(p->maps);
queue_head_t *mappool = &(p->map_pool);
queue_head_t *mapwaste = &(p->map_waste);
pmap_pgtrace_map_t *map;
while (!queue_empty(mapq)) {
queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain);
queue_enter_first(mapwaste, map, pmap_pgtrace_map_t*, chain);
}
PMAP_PGTRACE_UNLOCK(&ints);
queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) {
if (map->cloned == true) {
pmap_lock(map->pmap);
ptep = pmap_pte(map->pmap, map->ova);
assert(ptep);
PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID);
PMAP_UPDATE_TLBS(map->pmap, map->ova, map->ova + ARM_PGBYTES, false);
for (int i = 0; i < 3; i++) {
ptep = pmap_pte(kernel_pmap, map->cva[i]);
assert(ptep != NULL);
PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]);
PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i] + ARM_PGBYTES, false);
}
pmap_unlock(map->pmap);
}
map->pmap = NULL;
map->ova = (vm_map_offset_t)NULL;
map->cloned = false;
}
PMAP_PGTRACE_LOCK(&ints);
while (!queue_empty(mapwaste)) {
queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain);
queue_enter_first(mappool, map, pmap_pgtrace_map_t*, chain);
}
PMAP_PGTRACE_UNLOCK(&ints);
}
inline static void
pmap_pgtrace_get_search_space(pmap_t pmap, vm_map_offset_t *startp, vm_map_offset_t *endp)
{
uint64_t tsz;
vm_map_offset_t end;
if (pmap == kernel_pmap) {
tsz = (get_tcr() >> TCR_T1SZ_SHIFT) & TCR_TSZ_MASK;
*startp = MAX(VM_MIN_KERNEL_ADDRESS, (UINT64_MAX >> (64 - tsz)) << (64 - tsz));
*endp = VM_MAX_KERNEL_ADDRESS;
} else {
tsz = (get_tcr() >> TCR_T0SZ_SHIFT) & TCR_TSZ_MASK;
if (tsz == 64) {
end = 0;
} else {
end = ((uint64_t)1 << (64 - tsz)) - 1;
}
*startp = 0;
*endp = end;
}
assert(*endp > *startp);
return;
}
static uint64_t
pmap_pgtrace_clone_from_pa(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset)
{
uint64_t ret = 0;
vm_map_offset_t min, max;
vm_map_offset_t cur_page, end_page;
pt_entry_t *ptep;
tt_entry_t *ttep;
tt_entry_t tte;
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
pmap_pgtrace_get_search_space(pmap, &min, &max);
cur_page = arm_trunc_page(min);
end_page = arm_trunc_page(max);
while (cur_page <= end_page) {
vm_map_offset_t add = 0;
pmap_lock(pmap);
if (pmap == kernel_pmap &&
((vm_kernel_base <= cur_page && cur_page < vm_kernel_top) ||
(vm_kext_base <= cur_page && cur_page < vm_kext_top))) {
add = ARM_PGBYTES;
goto unlock_continue;
}
ttep = pmap_tt1e(pmap, cur_page);
assert(ttep);
tte = *ttep;
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
add = ARM_TT_L1_SIZE;
goto unlock_continue;
}
tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, pt_attr, cur_page)];
if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) {
add = ARM_TT_L2_SIZE;
goto unlock_continue;
}
ptep = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, pt_attr, cur_page)]);
if (ptep == PT_ENTRY_NULL) {
add = ARM_TT_L3_SIZE;
goto unlock_continue;
}
if (arm_trunc_page(pa) == pte_to_pa(*ptep)) {
if (pmap_pgtrace_enter_clone(pmap, cur_page, start_offset, end_offset) == true) {
ret++;
}
}
add = ARM_PGBYTES;
unlock_continue:
pmap_unlock(pmap);
if (cur_page + add < cur_page) {
break;
}
cur_page += add;
}
return ret;
}
static uint64_t
pmap_pgtrace_clone_from_pvtable(pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset)
{
uint64_t ret = 0;
unsigned long pai;
pv_entry_t **pvh;
pt_entry_t *ptep;
pmap_t pmap;
queue_head_t pmapvaq;
pmap_va_t *pmapva;
queue_init(&pmapvaq);
pai = pa_index(pa);
LOCK_PVH(pai);
pvh = pai_to_pvh(pai);
if (pvh_test_type(pvh, PVH_TYPE_PTEP)) {
ptep = pvh_ptep(pvh);
pmap = ptep_get_pmap(ptep);
pmapva = (pmap_va_t *)zalloc(ZV_PMAP_VA);
pmapva->pmap = pmap;
pmapva->va = ptep_get_va(ptep);
queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain);
} else if (pvh_test_type(pvh, PVH_TYPE_PVEP)) {
pv_entry_t *pvep;
pvep = pvh_list(pvh);
while (pvep) {
ptep = pve_get_ptep(pvep);
pmap = ptep_get_pmap(ptep);
pmapva = (pmap_va_t *)zalloc(ZV_PMAP_VA);
pmapva->pmap = pmap;
pmapva->va = ptep_get_va(ptep);
queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain);
pvep = PVE_NEXT_PTR(pve_next(pvep));
}
}
UNLOCK_PVH(pai);
queue_iterate(&pmapvaq, pmapva, pmap_va_t *, chain) {
pmap_lock(pmapva->pmap);
ptep = pmap_pte(pmapva->pmap, pmapva->va);
if (pte_to_pa(*ptep) == pa) {
if (pmap_pgtrace_enter_clone(pmapva->pmap, pmapva->va, start_offset, end_offset) == true) {
ret++;
}
}
pmap_unlock(pmapva->pmap);
zfree(ZV_PMAP_VA, pmapva);
}
return ret;
}
static pmap_pgtrace_page_t *
pmap_pgtrace_alloc_page(void)
{
pmap_pgtrace_page_t *p;
queue_head_t *mapq;
queue_head_t *mappool;
queue_head_t *mapwaste;
pmap_pgtrace_map_t *map;
p = zalloc(ZV_PMAP_PGTRACE);
assert(p);
p->state = UNDEFINED;
mapq = &(p->maps);
mappool = &(p->map_pool);
mapwaste = &(p->map_waste);
queue_init(mapq);
queue_init(mappool);
queue_init(mapwaste);
for (int i = 0; i < PGTRACE_MAX_MAP; i++) {
vm_map_offset_t newcva;
pt_entry_t *cptep;
kern_return_t kr;
vm_map_entry_t entry;
vm_object_reference(kernel_object);
kr = vm_map_find_space(kernel_map, &newcva, vm_map_round_page(3 * ARM_PGBYTES, PAGE_MASK), 0, 0, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_DIAG, &entry);
if (kr != KERN_SUCCESS) {
panic("%s VM couldn't find any space kr=%d\n", __func__, kr);
}
VME_OBJECT_SET(entry, kernel_object);
VME_OFFSET_SET(entry, newcva);
vm_map_unlock(kernel_map);
map = zalloc(ZV_PMAP_PGTRACE);
for (int j = 0; j < 3; j++) {
vm_map_offset_t addr = newcva + j * ARM_PGBYTES;
kr = pmap_expand(kernel_pmap, addr, 0, PMAP_TT_L3_LEVEL);
if (kr != KERN_SUCCESS) {
panic("%s: pmap_expand(kernel_pmap, addr=%llx) returns kr=%d\n", __func__, addr, kr);
}
cptep = pmap_pte(kernel_pmap, addr);
assert(cptep != NULL);
map->cva[j] = addr;
map->cva_spte[j] = *cptep;
}
map->range.start = map->range.end = 0;
map->cloned = false;
queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain);
}
return p;
}
static void
pmap_pgtrace_free_page(pmap_pgtrace_page_t *p)
{
queue_head_t *mapq;
queue_head_t *mappool;
queue_head_t *mapwaste;
pmap_pgtrace_map_t *map;
assert(p);
mapq = &(p->maps);
mappool = &(p->map_pool);
mapwaste = &(p->map_waste);
while (!queue_empty(mapq)) {
queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain);
zfree(ZV_PMAP_PGTRACE, map);
}
while (!queue_empty(mappool)) {
queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain);
zfree(ZV_PMAP_PGTRACE, map);
}
while (!queue_empty(mapwaste)) {
queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain);
zfree(ZV_PMAP_PGTRACE, map);
}
zfree(ZV_PMAP_PGTRACE, p);
}
int
pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end)
{
int ret = 0;
pt_entry_t *ptep;
queue_head_t *q = &(pmap_pgtrace.pages);
uint64_t ints;
vm_map_offset_t cur_page, end_page;
if (start > end) {
kprintf("%s: invalid start=%llx > end=%llx\n", __func__, start, end);
return -1;
}
PROF_START
cur_page = arm_trunc_page(start);
end_page = arm_trunc_page(end);
while (cur_page <= end_page) {
pmap_paddr_t pa_page = 0;
uint64_t num_cloned = 0;
pmap_pgtrace_page_t *p = NULL, *newp;
bool free_newp = true;
pmap_pgtrace_page_state_t state;
newp = pmap_pgtrace_alloc_page();
if (pmap != NULL) {
pmap_lock_ro(pmap);
}
if (pmap != kernel_pmap) {
pmap_lock_ro(kernel_pmap);
}
if (pmap == NULL) {
ptep = NULL;
pa_page = cur_page;
state = VA_UNDEFINED;
} else {
ptep = pmap_pte(pmap, cur_page);
if (ptep != NULL) {
pa_page = pte_to_pa(*ptep);
state = DEFINED;
} else {
state = PA_UNDEFINED;
}
}
PMAP_PGTRACE_LOCK(&ints);
if (state != PA_UNDEFINED) {
p = pmap_pgtrace_find_page(pa_page);
}
if (p == NULL) {
queue_enter_first(q, newp, pmap_pgtrace_page_t *, chain);
p = newp;
free_newp = false;
}
p->state = state;
queue_head_t *mapq = &(p->maps);
queue_head_t *mappool = &(p->map_pool);
pmap_pgtrace_map_t *map;
vm_map_offset_t start_offset, end_offset;
if (cur_page > start) {
start_offset = 0;
} else {
start_offset = start - cur_page;
}
if (cur_page == end_page) {
end_offset = end - end_page;
} else {
end_offset = ARM_PGBYTES - 1;
}
kprintf("%s: pmap=%p cur_page=%llx ptep=%p state=%d start_offset=%llx end_offset=%llx\n", __func__, pmap, cur_page, ptep, state, start_offset, end_offset);
assert(!queue_empty(mappool));
queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain);
if (p->state == PA_UNDEFINED) {
map->pmap = pmap;
map->ova = cur_page;
map->range.start = start_offset;
map->range.end = end_offset;
} else if (p->state == VA_UNDEFINED) {
p->pa = pa_page;
map->range.start = start_offset;
map->range.end = end_offset;
} else if (p->state == DEFINED) {
p->pa = pa_page;
map->pmap = pmap;
map->ova = cur_page;
map->range.start = start_offset;
map->range.end = end_offset;
} else {
panic("invalid p->state=%d\n", p->state);
}
map->cloned = false;
queue_enter(mapq, map, pmap_pgtrace_map_t *, chain);
PMAP_PGTRACE_UNLOCK(&ints);
if (pmap != kernel_pmap) {
pmap_unlock_ro(kernel_pmap);
}
if (pmap != NULL) {
pmap_unlock_ro(pmap);
}
if (pa_valid(pa_page)) {
num_cloned = pmap_pgtrace_clone_from_pvtable(pa_page, start_offset, end_offset);
}
if (pmap == NULL) {
num_cloned += pmap_pgtrace_clone_from_pa(kernel_pmap, pa_page, start_offset, end_offset);
} else {
num_cloned += pmap_pgtrace_clone_from_pa(pmap, pa_page, start_offset, end_offset);
}
if (free_newp) {
pmap_pgtrace_free_page(newp);
}
if (num_cloned == 0) {
kprintf("%s: no mapping found for pa_page=%llx but will be added when a page entered\n", __func__, pa_page);
}
ret += num_cloned;
if (cur_page + ARM_PGBYTES < cur_page) {
break;
} else {
cur_page += ARM_PGBYTES;
}
}
PROF_END
return ret;
}
int
pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end)
{
int ret = 0;
uint64_t ints;
queue_head_t *q = &(pmap_pgtrace.pages);
pmap_pgtrace_page_t *p;
vm_map_offset_t cur_page, end_page;
kprintf("%s start=%llx end=%llx\n", __func__, start, end);
PROF_START
pt_entry_t *ptep;
pmap_paddr_t pa_page;
cur_page = arm_trunc_page(start);
end_page = arm_trunc_page(end);
while (cur_page <= end_page) {
p = NULL;
if (pmap == NULL) {
pa_page = cur_page;
} else {
pmap_lock(pmap);
ptep = pmap_pte(pmap, cur_page);
if (ptep == NULL) {
pmap_unlock(pmap);
goto cont;
}
pa_page = pte_to_pa(*ptep);
pmap_unlock(pmap);
}
pmap_pgtrace_remove_all_clone(pa_page);
PMAP_PGTRACE_LOCK(&ints);
p = pmap_pgtrace_find_page(pa_page);
if (p != NULL) {
queue_remove(q, p, pmap_pgtrace_page_t *, chain);
ret++;
}
PMAP_PGTRACE_UNLOCK(&ints);
if (p != NULL) {
pmap_pgtrace_free_page(p);
}
cont:
if (cur_page + ARM_PGBYTES < cur_page) {
break;
} else {
cur_page += ARM_PGBYTES;
}
}
PROF_END
return ret;
}
kern_return_t
pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss)
{
pt_entry_t *ptep;
pgtrace_run_result_t res;
pmap_pgtrace_page_t *p;
uint64_t ints, found = false;
pmap_paddr_t pa;
ptep = pmap_pte(pmap, va);
if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) {
return KERN_FAILURE;
}
PMAP_PGTRACE_LOCK(&ints);
ptep = pmap_pte(pmap, va);
if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) {
PMAP_PGTRACE_UNLOCK(&ints);
return KERN_FAILURE;
} else if ((*ptep & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE_VALID) {
kprintf("%s Somehow this cpu's tlb has not updated?\n", __func__);
PMAP_UPDATE_TLBS(pmap, va, va + ARM_PGBYTES, false);
PMAP_PGTRACE_UNLOCK(&ints);
return KERN_SUCCESS;
}
pa = pte_to_pa(*ptep);
p = pmap_pgtrace_find_page(arm_trunc_page(pa));
if (p == NULL) {
panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa);
}
queue_head_t *mapq = &(p->maps);
queue_head_t *mapwaste = &(p->map_waste);
pmap_pgtrace_map_t *map;
queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) {
if (map->pmap == pmap && map->ova == arm_trunc_page(va)) {
found = true;
break;
}
}
if (!found) {
queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) {
if (map->pmap == pmap && map->ova == arm_trunc_page(va)) {
found = true;
break;
}
}
}
if (!found) {
panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa);
}
bzero(&res, sizeof(res));
pgtrace_decode_and_run(*(uint32_t *)get_saved_state_pc(ss), va, map->cva, ss, &res);
vm_map_offset_t offset = va - map->ova;
if (map->range.start <= offset && offset <= map->range.end) {
pgtrace_write_log(res);
}
PMAP_PGTRACE_UNLOCK(&ints);
add_saved_state_pc(ss, sizeof(uint32_t));
return KERN_SUCCESS;
}
#endif
uint64_t
pmap_shared_region_size_min(__unused pmap_t pmap)
{
#if (__ARM_VMSA__ > 7)
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
return pt_attr_twig_size(pt_attr);
#else
return ARM_NESTING_SIZE_MIN;
#endif
}
uint64_t
pmap_nesting_size_max(__unused pmap_t pmap)
{
return ARM_NESTING_SIZE_MAX;
}
boolean_t
pmap_enforces_execute_only(
#if (__ARM_VMSA__ == 7)
__unused
#endif
pmap_t pmap)
{
#if (__ARM_VMSA__ > 7)
return pmap != kernel_pmap;
#else
return FALSE;
#endif
}
MARK_AS_PMAP_TEXT void
pmap_set_vm_map_cs_enforced_internal(
pmap_t pmap,
bool new_value)
{
VALIDATE_PMAP(pmap);
pmap->pmap_vm_map_cs_enforced = new_value;
}
void
pmap_set_vm_map_cs_enforced(
pmap_t pmap,
bool new_value)
{
#if XNU_MONITOR
pmap_set_vm_map_cs_enforced_ppl(pmap, new_value);
#else
pmap_set_vm_map_cs_enforced_internal(pmap, new_value);
#endif
}
extern int cs_process_enforcement_enable;
bool
pmap_get_vm_map_cs_enforced(
pmap_t pmap)
{
if (cs_process_enforcement_enable) {
return true;
}
return pmap->pmap_vm_map_cs_enforced;
}
MARK_AS_PMAP_TEXT void
pmap_set_jit_entitled_internal(
__unused pmap_t pmap)
{
return;
}
void
pmap_set_jit_entitled(
pmap_t pmap)
{
#if XNU_MONITOR
pmap_set_jit_entitled_ppl(pmap);
#else
pmap_set_jit_entitled_internal(pmap);
#endif
}
bool
pmap_get_jit_entitled(
__unused pmap_t pmap)
{
return false;
}
MARK_AS_PMAP_TEXT static kern_return_t
pmap_query_page_info_internal(
pmap_t pmap,
vm_map_offset_t va,
int *disp_p)
{
pmap_paddr_t pa;
int disp;
int pai;
pt_entry_t *pte;
pv_entry_t **pv_h, *pve_p;
if (pmap == PMAP_NULL || pmap == kernel_pmap) {
pmap_pin_kernel_pages((vm_offset_t)disp_p, sizeof(*disp_p));
*disp_p = 0;
pmap_unpin_kernel_pages((vm_offset_t)disp_p, sizeof(*disp_p));
return KERN_INVALID_ARGUMENT;
}
disp = 0;
VALIDATE_PMAP(pmap);
pmap_lock_ro(pmap);
pte = pmap_pte(pmap, va);
if (pte == PT_ENTRY_NULL) {
goto done;
}
pa = pte_to_pa(*((volatile pt_entry_t*)pte));
if (pa == 0) {
if (ARM_PTE_IS_COMPRESSED(*pte, pte)) {
disp |= PMAP_QUERY_PAGE_COMPRESSED;
if (*pte & ARM_PTE_COMPRESSED_ALT) {
disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT;
}
}
} else {
disp |= PMAP_QUERY_PAGE_PRESENT;
pai = (int) pa_index(pa);
if (!pa_valid(pa)) {
goto done;
}
LOCK_PVH(pai);
pv_h = pai_to_pvh(pai);
pve_p = PV_ENTRY_NULL;
if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) {
pve_p = pvh_list(pv_h);
while (pve_p != PV_ENTRY_NULL &&
pve_get_ptep(pve_p) != pte) {
pve_p = PVE_NEXT_PTR(pve_next(pve_p));
}
}
if (IS_ALTACCT_PAGE(pai, pve_p)) {
disp |= PMAP_QUERY_PAGE_ALTACCT;
} else if (IS_REUSABLE_PAGE(pai)) {
disp |= PMAP_QUERY_PAGE_REUSABLE;
} else if (IS_INTERNAL_PAGE(pai)) {
disp |= PMAP_QUERY_PAGE_INTERNAL;
}
UNLOCK_PVH(pai);
}
done:
pmap_unlock_ro(pmap);
pmap_pin_kernel_pages((vm_offset_t)disp_p, sizeof(*disp_p));
*disp_p = disp;
pmap_unpin_kernel_pages((vm_offset_t)disp_p, sizeof(*disp_p));
return KERN_SUCCESS;
}
kern_return_t
pmap_query_page_info(
pmap_t pmap,
vm_map_offset_t va,
int *disp_p)
{
#if XNU_MONITOR
return pmap_query_page_info_ppl(pmap, va, disp_p);
#else
return pmap_query_page_info_internal(pmap, va, disp_p);
#endif
}
MARK_AS_PMAP_TEXT kern_return_t
pmap_return_internal(__unused boolean_t do_panic, __unused boolean_t do_recurse)
{
return KERN_SUCCESS;
}
kern_return_t
pmap_return(boolean_t do_panic, boolean_t do_recurse)
{
#if XNU_MONITOR
return pmap_return_ppl(do_panic, do_recurse);
#else
return pmap_return_internal(do_panic, do_recurse);
#endif
}
kern_return_t
pmap_load_legacy_trust_cache(struct pmap_legacy_trust_cache __unused *trust_cache,
const vm_size_t __unused trust_cache_len)
{
return KERN_NOT_SUPPORTED;
}
pmap_tc_ret_t
pmap_load_image4_trust_cache(struct pmap_image4_trust_cache __unused *trust_cache,
const vm_size_t __unused trust_cache_len,
uint8_t const * __unused img4_manifest,
const vm_size_t __unused img4_manifest_buffer_len,
const vm_size_t __unused img4_manifest_actual_len,
bool __unused dry_run)
{
return PMAP_TC_UNKNOWN_FORMAT;
}
bool
pmap_in_ppl(void)
{
return false;
}
void
pmap_lockdown_image4_slab(__unused vm_offset_t slab, __unused vm_size_t slab_len, __unused uint64_t flags)
{
}
void *
pmap_claim_reserved_ppl_page(void)
{
return NULL;
}
void
pmap_free_reserved_ppl_page(void __unused *kva)
{
}
MARK_AS_PMAP_TEXT static bool
pmap_is_trust_cache_loaded_internal(const uuid_t uuid)
{
bool found = false;
pmap_simple_lock(&pmap_loaded_trust_caches_lock);
for (struct pmap_image4_trust_cache const *c = pmap_image4_trust_caches; c != NULL; c = c->next) {
if (bcmp(uuid, c->module->uuid, sizeof(uuid_t)) == 0) {
found = true;
goto done;
}
}
#ifdef PLATFORM_BridgeOS
for (struct pmap_legacy_trust_cache const *c = pmap_legacy_trust_caches; c != NULL; c = c->next) {
if (bcmp(uuid, c->uuid, sizeof(uuid_t)) == 0) {
found = true;
goto done;
}
}
#endif
done:
pmap_simple_unlock(&pmap_loaded_trust_caches_lock);
return found;
}
bool
pmap_is_trust_cache_loaded(const uuid_t uuid)
{
#if XNU_MONITOR
return pmap_is_trust_cache_loaded_ppl(uuid);
#else
return pmap_is_trust_cache_loaded_internal(uuid);
#endif
}
MARK_AS_PMAP_TEXT static bool
pmap_lookup_in_loaded_trust_caches_internal(const uint8_t cdhash[CS_CDHASH_LEN])
{
struct pmap_image4_trust_cache const *cache = NULL;
#ifdef PLATFORM_BridgeOS
struct pmap_legacy_trust_cache const *legacy = NULL;
#endif
pmap_simple_lock(&pmap_loaded_trust_caches_lock);
for (cache = pmap_image4_trust_caches; cache != NULL; cache = cache->next) {
uint8_t hash_type = 0, flags = 0;
if (lookup_in_trust_cache_module(cache->module, cdhash, &hash_type, &flags)) {
goto done;
}
}
#ifdef PLATFORM_BridgeOS
for (legacy = pmap_legacy_trust_caches; legacy != NULL; legacy = legacy->next) {
for (uint32_t i = 0; i < legacy->num_hashes; i++) {
if (bcmp(legacy->hashes[i], cdhash, CS_CDHASH_LEN) == 0) {
goto done;
}
}
}
#endif
done:
pmap_simple_unlock(&pmap_loaded_trust_caches_lock);
if (cache != NULL) {
return true;
#ifdef PLATFORM_BridgeOS
} else if (legacy != NULL) {
return true;
#endif
}
return false;
}
bool
pmap_lookup_in_loaded_trust_caches(const uint8_t cdhash[CS_CDHASH_LEN])
{
#if XNU_MONITOR
return pmap_lookup_in_loaded_trust_caches_ppl(cdhash);
#else
return pmap_lookup_in_loaded_trust_caches_internal(cdhash);
#endif
}
MARK_AS_PMAP_TEXT static uint32_t
pmap_lookup_in_static_trust_cache_internal(const uint8_t cdhash[CS_CDHASH_LEN])
{
return lookup_in_static_trust_cache(cdhash);
}
uint32_t
pmap_lookup_in_static_trust_cache(const uint8_t cdhash[CS_CDHASH_LEN])
{
#if XNU_MONITOR
return pmap_lookup_in_static_trust_cache_ppl(cdhash);
#else
return pmap_lookup_in_static_trust_cache_internal(cdhash);
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_set_compilation_service_cdhash_internal(const uint8_t cdhash[CS_CDHASH_LEN])
{
pmap_simple_lock(&pmap_compilation_service_cdhash_lock);
memcpy(pmap_compilation_service_cdhash, cdhash, CS_CDHASH_LEN);
pmap_simple_unlock(&pmap_compilation_service_cdhash_lock);
pmap_cs_log("Added Compilation Service CDHash through the PPL: 0x%02X 0x%02X 0x%02X 0x%02X", cdhash[0], cdhash[1], cdhash[2], cdhash[4]);
}
MARK_AS_PMAP_TEXT static bool
pmap_match_compilation_service_cdhash_internal(const uint8_t cdhash[CS_CDHASH_LEN])
{
bool match = false;
pmap_simple_lock(&pmap_compilation_service_cdhash_lock);
if (bcmp(pmap_compilation_service_cdhash, cdhash, CS_CDHASH_LEN) == 0) {
match = true;
}
pmap_simple_unlock(&pmap_compilation_service_cdhash_lock);
if (match) {
pmap_cs_log("Matched Compilation Service CDHash through the PPL");
}
return match;
}
void
pmap_set_compilation_service_cdhash(const uint8_t cdhash[CS_CDHASH_LEN])
{
#if XNU_MONITOR
pmap_set_compilation_service_cdhash_ppl(cdhash);
#else
pmap_set_compilation_service_cdhash_internal(cdhash);
#endif
}
bool
pmap_match_compilation_service_cdhash(const uint8_t cdhash[CS_CDHASH_LEN])
{
#if XNU_MONITOR
return pmap_match_compilation_service_cdhash_ppl(cdhash);
#else
return pmap_match_compilation_service_cdhash_internal(cdhash);
#endif
}
MARK_AS_PMAP_TEXT static void
pmap_footprint_suspend_internal(
vm_map_t map,
boolean_t suspend)
{
#if DEVELOPMENT || DEBUG
if (suspend) {
current_thread()->pmap_footprint_suspended = TRUE;
map->pmap->footprint_was_suspended = TRUE;
} else {
current_thread()->pmap_footprint_suspended = FALSE;
}
#else
(void) map;
(void) suspend;
#endif
}
void
pmap_footprint_suspend(
vm_map_t map,
boolean_t suspend)
{
#if XNU_MONITOR
pmap_footprint_suspend_ppl(map, suspend);
#else
pmap_footprint_suspend_internal(map, suspend);
#endif
}
#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
struct page_table_dump_header {
uint64_t pa;
uint64_t num_entries;
uint64_t start_va;
uint64_t end_va;
};
static kern_return_t
pmap_dump_page_tables_recurse(pmap_t pmap,
const tt_entry_t *ttp,
unsigned int cur_level,
unsigned int level_mask,
uint64_t start_va,
void *buf_start,
void *buf_end,
size_t *bytes_copied)
{
const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
uint64_t num_entries = pt_attr_page_size(pt_attr) / sizeof(*ttp);
uint64_t size = pt_attr->pta_level_info[cur_level].size;
uint64_t valid_mask = pt_attr->pta_level_info[cur_level].valid_mask;
uint64_t type_mask = pt_attr->pta_level_info[cur_level].type_mask;
uint64_t type_block = pt_attr->pta_level_info[cur_level].type_block;
void *bufp = (uint8_t*)buf_start + *bytes_copied;
if (cur_level == pt_attr_root_level(pt_attr)) {
num_entries = pmap_root_alloc_size(pmap) / sizeof(tt_entry_t);
}
uint64_t tt_size = num_entries * sizeof(tt_entry_t);
const tt_entry_t *tt_end = &ttp[num_entries];
if (((vm_offset_t)buf_end - (vm_offset_t)bufp) < (tt_size + sizeof(struct page_table_dump_header))) {
return KERN_INSUFFICIENT_BUFFER_SIZE;
}
if (level_mask & (1U << cur_level)) {
struct page_table_dump_header *header = (struct page_table_dump_header*)bufp;
header->pa = ml_static_vtop((vm_offset_t)ttp);
header->num_entries = num_entries;
header->start_va = start_va;
header->end_va = start_va + (num_entries * size);
bcopy(ttp, (uint8_t*)bufp + sizeof(*header), tt_size);
*bytes_copied = *bytes_copied + sizeof(*header) + tt_size;
}
uint64_t current_va = start_va;
for (const tt_entry_t *ttep = ttp; ttep < tt_end; ttep++, current_va += size) {
tt_entry_t tte = *ttep;
if (!(tte & valid_mask)) {
continue;
}
if ((tte & type_mask) == type_block) {
continue;
} else {
if (cur_level >= pt_attr_leaf_level(pt_attr)) {
panic("%s: corrupt entry %#llx at %p, "
"ttp=%p, cur_level=%u, bufp=%p, buf_end=%p",
__FUNCTION__, tte, ttep,
ttp, cur_level, bufp, buf_end);
}
const tt_entry_t *next_tt = (const tt_entry_t*)phystokv(tte & ARM_TTE_TABLE_MASK);
kern_return_t recurse_result = pmap_dump_page_tables_recurse(pmap, next_tt, cur_level + 1,
level_mask, current_va, buf_start, buf_end, bytes_copied);
if (recurse_result != KERN_SUCCESS) {
return recurse_result;
}
}
}
return KERN_SUCCESS;
}
kern_return_t
pmap_dump_page_tables(pmap_t pmap, void *bufp, void *buf_end, unsigned int level_mask, size_t *bytes_copied)
{
if (not_in_kdp) {
panic("pmap_dump_page_tables must only be called from kernel debugger context");
}
return pmap_dump_page_tables_recurse(pmap, pmap->tte, pt_attr_root_level(pmap_get_pt_attr(pmap)),
level_mask, pmap->min, bufp, buf_end, bytes_copied);
}
#else
kern_return_t
pmap_dump_page_tables(pmap_t pmap __unused, void *bufp __unused, void *buf_end __unused,
unsigned int level_mask __unused, size_t *bytes_copied __unused)
{
return KERN_NOT_SUPPORTED;
}
#endif
#ifdef CONFIG_XNUPOST
#ifdef __arm64__
static volatile bool pmap_test_took_fault = false;
static bool
pmap_test_fault_handler(arm_saved_state_t * state)
{
bool retval = false;
uint32_t esr = get_saved_state_esr(state);
esr_exception_class_t class = ESR_EC(esr);
fault_status_t fsc = ISS_IA_FSC(ESR_ISS(esr));
if ((class == ESR_EC_DABORT_EL1) &&
((fsc == FSC_PERMISSION_FAULT_L3) || (fsc == FSC_ACCESS_FLAG_FAULT_L3))) {
pmap_test_took_fault = true;
set_saved_state_pc(state, get_saved_state_pc(state) + 4);
retval = true;
}
return retval;
}
static bool
pmap_test_access(pmap_t pmap, vm_map_address_t va, bool should_fault, bool is_write)
{
uint64_t old_int_state = pmap_interrupts_disable();
pmap_t old_pmap = current_pmap();
mp_disable_preemption();
pmap_switch(pmap);
pmap_test_took_fault = false;
#if __ARM_PAN_AVAILABLE__
__builtin_arm_wsr("pan", 0);
#endif
ml_expect_fault_begin(pmap_test_fault_handler, va);
if (is_write) {
*((volatile uint64_t*)(va)) = 0xdec0de;
} else {
volatile uint64_t tmp = *((volatile uint64_t*)(va));
(void)tmp;
}
bool took_fault = pmap_test_took_fault;
ml_expect_fault_end();
#if __ARM_PAN_AVAILABLE__
__builtin_arm_wsr("pan", 1);
#endif
pmap_switch(old_pmap);
mp_enable_preemption();
pmap_interrupts_restore(old_int_state);
bool retval = (took_fault == should_fault);
return retval;
}
static bool
pmap_test_read(pmap_t pmap, vm_map_address_t va, bool should_fault)
{
bool retval = pmap_test_access(pmap, va, should_fault, false);
if (!retval) {
T_FAIL("%s: %s, "
"pmap=%p, va=%p, should_fault=%u",
__func__, should_fault ? "did not fault" : "faulted",
pmap, (void*)va, (unsigned)should_fault);
}
return retval;
}
static bool
pmap_test_write(pmap_t pmap, vm_map_address_t va, bool should_fault)
{
bool retval = pmap_test_access(pmap, va, should_fault, true);
if (!retval) {
T_FAIL("%s: %s, "
"pmap=%p, va=%p, should_fault=%u",
__func__, should_fault ? "did not fault" : "faulted",
pmap, (void*)va, (unsigned)should_fault);
}
return retval;
}
static bool
pmap_test_check_refmod(pmap_paddr_t pa, unsigned int should_be_set)
{
unsigned int should_be_clear = (~should_be_set) & (VM_MEM_REFERENCED | VM_MEM_MODIFIED);
unsigned int bits = pmap_get_refmod((ppnum_t)atop(pa));
bool retval = (((bits & should_be_set) == should_be_set) && ((bits & should_be_clear) == 0));
if (!retval) {
T_FAIL("%s: bits=%u, "
"pa=%p, should_be_set=%u",
__func__, bits,
(void*)pa, should_be_set);
}
return retval;
}
static __attribute__((noinline)) bool
pmap_test_read_write(pmap_t pmap, vm_map_address_t va, bool allow_read, bool allow_write)
{
bool retval = (pmap_test_read(pmap, va, !allow_read) | pmap_test_write(pmap, va, !allow_write));
return retval;
}
static int
pmap_test_test_config(unsigned int flags)
{
T_LOG("running pmap_test_test_config flags=0x%X", flags);
unsigned int map_count = 0;
unsigned long page_ratio = 0;
pmap_t pmap = pmap_create_options(NULL, 0, flags);
if (!pmap) {
panic("Failed to allocate pmap");
}
__unused const pt_attr_t * const pt_attr = pmap_get_pt_attr(pmap);
uintptr_t native_page_size = pt_attr_page_size(native_pt_attr);
uintptr_t pmap_page_size = pt_attr_page_size(pt_attr);
uintptr_t pmap_twig_size = pt_attr_twig_size(pt_attr);
if (pmap_page_size <= native_page_size) {
page_ratio = native_page_size / pmap_page_size;
} else {
panic("%s: page_ratio < 1, native_page_size=%lu, pmap_page_size=%lu"
"flags=%u",
__func__, native_page_size, pmap_page_size,
flags);
}
if (PAGE_RATIO > 1) {
pmap_page_size = PAGE_SIZE;
native_page_size = PAGE_SIZE;
}
vm_page_t unwired_vm_page = vm_page_grab();
vm_page_t wired_vm_page = vm_page_grab();
if ((unwired_vm_page == VM_PAGE_NULL) || (wired_vm_page == VM_PAGE_NULL)) {
panic("Failed to grab VM pages");
}
ppnum_t pn = VM_PAGE_GET_PHYS_PAGE(unwired_vm_page);
ppnum_t wired_pn = VM_PAGE_GET_PHYS_PAGE(wired_vm_page);
pmap_paddr_t pa = ptoa(pn);
pmap_paddr_t wired_pa = ptoa(wired_pn);
vm_map_address_t va_base = pmap_twig_size;
vm_map_address_t wired_va_base = ((2 * pmap_twig_size) - pmap_page_size);
if (wired_va_base < (va_base + (page_ratio * pmap_page_size))) {
panic("Cannot pin translation table");
}
pmap_enter_addr(pmap, wired_va_base, wired_pa, VM_PROT_READ, VM_PROT_READ, 0, true);
for (map_count = 0; map_count < page_ratio; map_count++) {
pmap_enter_addr(pmap, va_base + (pmap_page_size * map_count), pa + (pmap_page_size * (map_count)), VM_PROT_READ, VM_PROT_READ, 0, false);
}
for (map_count = 0; map_count < page_ratio; map_count++) {
pt_entry_t * ptep = pmap_pte(pmap, va_base + (pmap_page_size * map_count));
if (pte_to_pa(*ptep) != (pa + (pmap_page_size * map_count))) {
T_FAIL("Unexpected pa=%p, expected %p, map_count=%u",
(void*)pte_to_pa(*ptep), (void*)(pa + (pmap_page_size * map_count)), map_count);
}
if (ptep_get_va(ptep) != (va_base + (pmap_page_size * map_count))) {
T_FAIL("Unexpected va=%p, expected %p, map_count=%u",
(void*)ptep_get_va(ptep), (void*)(va_base + (pmap_page_size * map_count)), map_count);
}
}
T_LOG("Validate that reads to our mapping do not fault.");
pmap_test_read(pmap, va_base, false);
T_LOG("Validate that writes to our mapping fault.");
pmap_test_write(pmap, va_base, true);
T_LOG("Make the first mapping writable.");
pmap_enter_addr(pmap, va_base, pa, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0, false);
T_LOG("Validate that writes to our mapping do not fault.");
pmap_test_write(pmap, va_base, false);
T_LOG("Make the first mapping XO.");
pmap_enter_addr(pmap, va_base, pa, VM_PROT_EXECUTE, VM_PROT_EXECUTE, 0, false);
T_LOG("Validate that reads to our mapping do not fault.");
pmap_test_read(pmap, va_base, false);
T_LOG("Validate that writes to our mapping fault.");
pmap_test_write(pmap, va_base, true);
for (map_count = 1; map_count < page_ratio; map_count++) {
pmap_test_write(pmap, va_base + (pmap_page_size * map_count), true);
pmap_remove(pmap, va_base + (pmap_page_size * map_count), va_base + (pmap_page_size * map_count) + pmap_page_size);
}
T_LOG("Mark the page unreferenced and unmodified.");
pmap_clear_refmod(pn, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
pmap_test_check_refmod(pa, 0);
T_LOG("!ref/!mod: read, no fault. Expect ref/!mod");
pmap_enter_addr(pmap, va_base, pa, VM_PROT_READ, VM_PROT_NONE, 0, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED);
T_LOG("!ref/!mod: read, read fault. Expect ref/!mod");
pmap_clear_refmod(pn, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
pmap_enter_addr(pmap, va_base, pa, VM_PROT_READ, VM_PROT_READ, 0, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED);
T_LOG("!ref/!mod: rw, read fault. Expect ref/!mod");
pmap_clear_refmod(pn, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
pmap_enter_addr(pmap, va_base, pa, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED);
T_LOG("ref/!mod: rw, read fault. Expect ref/!mod");
pmap_enter_addr(pmap, va_base, pa, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ, 0, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED);
T_LOG("!ref/!mod: rw, rw fault. Expect ref/mod");
pmap_clear_refmod(pn, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
pmap_enter_addr(pmap, va_base, pa, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED | VM_MEM_MODIFIED);
vm_map_address_t rw_base = va_base;
vm_map_address_t ro_base = va_base + pmap_page_size;
pmap_enter_addr(pmap, rw_base, pa, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0, false);
pmap_enter_addr(pmap, ro_base, pa, VM_PROT_READ, VM_PROT_READ, 0, false);
T_LOG("!ref/!mod: expect no access");
pmap_clear_refmod(pn, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
pmap_test_read_write(pmap, ro_base, false, false);
pmap_test_read_write(pmap, rw_base, false, false);
T_LOG("Read fault; expect !ref/!mod -> ref/!mod, read access");
arm_fast_fault(pmap, rw_base, VM_PROT_READ, false, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, false);
T_LOG("Write fault; expect ref/!mod -> ref/mod, read and write access");
arm_fast_fault(pmap, rw_base, VM_PROT_READ | VM_PROT_WRITE, false, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED | VM_MEM_MODIFIED);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, true);
T_LOG("Write fault; expect !ref/!mod -> ref/mod, read and write access");
pmap_clear_refmod(pn, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
arm_fast_fault(pmap, rw_base, VM_PROT_READ | VM_PROT_WRITE, false, false);
pmap_test_check_refmod(pa, VM_MEM_REFERENCED | VM_MEM_MODIFIED);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, true);
T_LOG("RW protect both mappings; should not change protections.");
pmap_protect(pmap, ro_base, ro_base + pmap_page_size, VM_PROT_READ | VM_PROT_WRITE);
pmap_protect(pmap, rw_base, rw_base + pmap_page_size, VM_PROT_READ | VM_PROT_WRITE);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, true);
T_LOG("Read protect both mappings; RW mapping should become RO.");
pmap_protect(pmap, ro_base, ro_base + pmap_page_size, VM_PROT_READ);
pmap_protect(pmap, rw_base, rw_base + pmap_page_size, VM_PROT_READ);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, false);
T_LOG("RW protect the page; mappings should not change protections.");
pmap_enter_addr(pmap, rw_base, pa, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0, false);
pmap_page_protect(pn, VM_PROT_ALL);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, true);
T_LOG("Read protect the page; RW mapping should become RO.");
pmap_page_protect(pn, VM_PROT_READ);
pmap_test_read_write(pmap, ro_base, true, false);
pmap_test_read_write(pmap, rw_base, true, false);
T_LOG("Validate that disconnect removes all known mappings of the page.");
pmap_disconnect(pn);
if (!pmap_verify_free(pn)) {
T_FAIL("Page still has mappings");
}
T_LOG("Remove the wired mapping, so we can tear down the test map.");
pmap_remove(pmap, wired_va_base, wired_va_base + pmap_page_size);
pmap_destroy(pmap);
T_LOG("Release the pages back to the VM.");
vm_page_lock_queues();
vm_page_free(unwired_vm_page);
vm_page_free(wired_vm_page);
vm_page_unlock_queues();
T_LOG("Testing successful!");
return 0;
}
#endif
kern_return_t
pmap_test(void)
{
T_LOG("Starting pmap_tests");
#ifdef __arm64__
int flags = 0;
flags |= PMAP_CREATE_64BIT;
#if __ARM_MIXED_PAGE_SIZE__
T_LOG("Testing VM_PAGE_SIZE_4KB");
pmap_test_test_config(flags | PMAP_CREATE_FORCE_4K_PAGES);
T_LOG("Testing VM_PAGE_SIZE_16KB");
pmap_test_test_config(flags);
#else
pmap_test_test_config(flags);
#endif
#endif
T_PASS("completed pmap_test successfully");
return KERN_SUCCESS;
}
#endif
#if DEVELOPMENT || DEBUG
#define ARM_UNDEFINED_INSN 0xe7f000f0
#define ARM_UNDEFINED_INSN_THUMB 0xde00
kern_return_t
pmap_test_text_corruption(pmap_paddr_t pa)
{
#if XNU_MONITOR
return pmap_test_text_corruption_ppl(pa);
#else
return pmap_test_text_corruption_internal(pa);
#endif
}
MARK_AS_PMAP_TEXT kern_return_t
pmap_test_text_corruption_internal(pmap_paddr_t pa)
{
vm_offset_t va = phystokv(pa);
unsigned int pai = pa_index(pa);
assert(pa_valid(pa));
LOCK_PVH(pai);
pv_entry_t **pv_h = pai_to_pvh(pai);
assert(!pvh_test_type(pv_h, PVH_TYPE_NULL));
#if defined(PVH_FLAG_EXEC)
const bool need_ap_twiddle = pvh_get_flags(pv_h) & PVH_FLAG_EXEC;
if (need_ap_twiddle) {
pmap_set_ptov_ap(pai, AP_RWNA, FALSE);
}
#endif
if (va & 1) {
va &= ~(vm_offset_t)1;
*(uint16_t *)va = ARM_UNDEFINED_INSN_THUMB;
} else {
*(uint32_t *)va = ARM_UNDEFINED_INSN;
}
#if defined(PVH_FLAG_EXEC)
if (need_ap_twiddle) {
pmap_set_ptov_ap(pai, AP_RONA, FALSE);
}
#endif
InvalidatePoU_IcacheRegion(va, sizeof(uint32_t));
UNLOCK_PVH(pai);
return KERN_SUCCESS;
}
#endif