#include <kern/zalloc.h>
#include <kern/kalloc.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <libkern/libkern.h>
ZONE_VIEW_DEFINE(ZV_NAMEI, "vfs.namei", KHEAP_ID_DATA_BUFFERS, MAXPATHLEN);
static void *
__MALLOC_ext(
size_t size,
int type,
int flags,
vm_allocation_site_t *site,
kalloc_heap_t heap)
{
void *addr = NULL;
if (type >= M_LAST) {
panic("_malloc TYPE");
}
if (size == 0) {
return NULL;
}
static_assert(sizeof(vm_size_t) == sizeof(size_t));
static_assert(M_WAITOK == Z_WAITOK);
static_assert(M_NOWAIT == Z_NOWAIT);
static_assert(M_ZERO == Z_ZERO);
addr = kalloc_ext(heap, size,
flags & (M_WAITOK | M_NOWAIT | M_ZERO), site).addr;
if (__probable(addr)) {
return addr;
}
if (flags & (M_NOWAIT | M_NULL)) {
return NULL;
}
panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
}
void *
__MALLOC(size_t size, int type, int flags, vm_allocation_site_t *site)
{
return __MALLOC_ext(size, type, flags, site, KHEAP_DEFAULT);
}
void *
__REALLOC(
void *addr,
size_t size,
int type __unused,
int flags,
vm_allocation_site_t *site)
{
addr = kheap_realloc_addr(KHEAP_DEFAULT, addr, size,
flags & (M_WAITOK | M_NOWAIT | M_ZERO), site).addr;
if (__probable(addr)) {
return addr;
}
if (flags & (M_NOWAIT | M_NULL)) {
return NULL;
}
panic("_REALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
}
void *
_MALLOC_external(size_t size, int type, int flags);
void *
_MALLOC_external(size_t size, int type, int flags)
{
static vm_allocation_site_t site = {
.tag = VM_KERN_MEMORY_KALLOC,
.flags = VM_TAG_BT,
};
return __MALLOC_ext(size, type, flags, &site, KHEAP_KEXT);
}
void
_FREE_external(void *addr, int type);
void
_FREE_external(void *addr, int type __unused)
{
kheap_free_addr(KHEAP_ANY, addr);
}
void
_FREE_ZONE_external(void *elem, size_t size, int type);
void
_FREE_ZONE_external(void *elem, size_t size, int type __unused)
{
(kheap_free)(KHEAP_KEXT, elem, size);
}
#if DEBUG || DEVELOPMENT
extern unsigned int zone_map_jetsam_limit;
static int
sysctl_zone_map_jetsam_limit SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
int oldval = 0, val = 0, error = 0;
oldval = zone_map_jetsam_limit;
error = sysctl_io_number(req, oldval, sizeof(int), &val, NULL);
if (error || !req->newptr) {
return error;
}
if (val <= 0 || val > 100) {
printf("sysctl_zone_map_jetsam_limit: new jetsam limit value is invalid.\n");
return EINVAL;
}
zone_map_jetsam_limit = val;
return 0;
}
SYSCTL_PROC(_kern, OID_AUTO, zone_map_jetsam_limit, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
sysctl_zone_map_jetsam_limit, "I", "Zone map jetsam limit");
extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
static int
sysctl_zone_map_size_and_capacity SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
uint64_t zstats[2];
get_zone_map_size(&zstats[0], &zstats[1]);
return SYSCTL_OUT(req, &zstats, sizeof(zstats));
}
SYSCTL_PROC(_kern, OID_AUTO, zone_map_size_and_capacity,
CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED,
0, 0, &sysctl_zone_map_size_and_capacity, "Q", "Current size and capacity of the zone map");
extern boolean_t run_zone_test(void);
static int
sysctl_run_zone_test SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
if (!req->newptr) {
return 0;
}
int ret_val = run_zone_test();
return SYSCTL_OUT(req, &ret_val, sizeof(ret_val));
}
SYSCTL_PROC(_kern, OID_AUTO, run_zone_test,
CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_MASKED | CTLFLAG_LOCKED,
0, 0, &sysctl_run_zone_test, "I", "Test zone allocator KPI");
#endif
#if CONFIG_ZLEAKS
SYSCTL_DECL(_kern_zleak);
SYSCTL_NODE(_kern, OID_AUTO, zleak, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "zleak");
static int
sysctl_zleak_active SYSCTL_HANDLER_ARGS
{
#pragma unused(arg1, arg2)
int oldval, val, error;
val = oldval = get_zleak_state();
error = sysctl_handle_int(oidp, &val, 0, req);
if (error || !req->newptr) {
return error;
}
if (val == 1 && oldval == 0) {
kern_return_t kr = zleak_activate();
if (KERN_SUCCESS != kr) {
printf("zleak_active: failed to activate "
"live zone leak debugging (%d).\n", kr);
}
}
if (val == 0 && oldval == 1) {
printf("zleak_active: active, cannot be disabled.\n");
return EINVAL;
}
return 0;
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, active,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, sysctl_zleak_active, "I", "zleak activity");
static int
sysctl_zleak_max_zonemap_size SYSCTL_HANDLER_ARGS
{
uint64_t zmap_max_size = *(vm_size_t *)arg1;
return sysctl_handle_quad(oidp, &zmap_max_size, arg2, req);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, max_zonemap_size,
CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED,
&zleak_max_zonemap_size, 0,
sysctl_zleak_max_zonemap_size, "Q", "zleak max zonemap size");
static int
sysctl_zleak_threshold SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg2)
int error;
uint64_t value = *(vm_size_t *)arg1;
error = sysctl_io_number(req, value, sizeof(value), &value, NULL);
if (error || !req->newptr) {
return error;
}
if (value > (uint64_t)zleak_max_zonemap_size) {
return ERANGE;
}
*(vm_size_t *)arg1 = value;
return 0;
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, global_threshold,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
&zleak_global_tracking_threshold, 0,
sysctl_zleak_threshold, "Q", "zleak global threshold");
SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
&zleak_per_zone_tracking_threshold, 0,
sysctl_zleak_threshold, "Q", "zleak per-zone threshold");
#endif
extern uint64_t get_zones_collectable_bytes(void);
static int
sysctl_zones_collectable_bytes SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
uint64_t zones_free_mem = get_zones_collectable_bytes();
return SYSCTL_OUT(req, &zones_free_mem, sizeof(zones_free_mem));
}
SYSCTL_PROC(_kern, OID_AUTO, zones_collectable_bytes,
CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED,
0, 0, &sysctl_zones_collectable_bytes, "Q", "Collectable memory in zones");
#if DEBUG || DEVELOPMENT
static int
sysctl_zone_gc_replenish_test SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
if (!req->newptr) {
return 0;
}
int ret_val = 0;
zone_gc_replenish_test();
return SYSCTL_OUT(req, &ret_val, sizeof(ret_val));
}
static int
sysctl_zone_alloc_replenish_test SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
if (!req->newptr) {
return 0;
}
int ret_val = 0;
zone_alloc_replenish_test();
return SYSCTL_OUT(req, &ret_val, sizeof(ret_val));
}
SYSCTL_PROC(_kern, OID_AUTO, zone_gc_replenish_test,
CTLTYPE_INT | CTLFLAG_MASKED | CTLFLAG_LOCKED | CTLFLAG_WR,
0, 0, &sysctl_zone_gc_replenish_test, "I", "Test zone GC replenish");
SYSCTL_PROC(_kern, OID_AUTO, zone_alloc_replenish_test,
CTLTYPE_INT | CTLFLAG_MASKED | CTLFLAG_LOCKED | CTLFLAG_WR,
0, 0, &sysctl_zone_alloc_replenish_test, "I", "Test zone alloc replenish");
#endif