#include <sys/param.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <net/route.h>
#include <net/necp.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_pcb.h>
#include <netinet/flow_divert.h>
#include <sys/event.h>
#include <sys/eventvar.h>
#include <sys/proc_internal.h>
#include <sys/mount_internal.h>
#include <sys/vnode_internal.h>
#include <sys/ubc_internal.h>
#include <sys/namei.h>
#include <sys/file_internal.h>
#include <sys/filedesc.h>
#include <sys/tty.h>
#include <sys/quota.h>
#include <sys/uio_internal.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <sys/decmpfs.h>
#include <miscfs/specfs/specdev.h>
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
#include <nfs/nfsnode.h>
#include <nfs/nfsmount.h>
#include <mach/mach_types.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
void kmeminit(void);
const char *memname[] = {
"free",
"mbuf",
"devbuf",
"socket",
"pcb",
"routetbl",
"hosttbl",
"fragtbl",
"zombie",
"ifaddr",
"soopts",
"soname",
"namei",
"gprof",
"ioctlops",
"mapmem",
"cred",
"pgrp",
"session",
"iov32",
"mount",
"fhandle",
#if (NFSCLIENT || NFSSERVER)
"NFS req",
"NFS mount",
"NFS node",
#else
"",
"",
"",
#endif
"vnodes",
"namecache",
#if QUOTA
"UFS quota",
#else
"",
#endif
"proc uuid policy",
#if (SYSV_SEM || SYSV_MSG || SYSV_SHM)
"shm",
#else
"",
#endif
"plimit",
"sigacts",
"VM object",
"VM objhash",
"VM pmap",
"VM pvmap",
"VM pager",
"VM pgdata",
"fileproc",
"file desc",
"lockf",
"proc",
"pstats",
"LFS segment",
"LFS node",
"",
"MFS node",
"NQNFS Lease",
"NQNFS Host",
"Export Host",
#if (NFSCLIENT || NFSSERVER)
"NFS srvsock",
"NFS uid",
"NFS daemon",
#else
"",
"",
"",
#endif
"ip_moptions",
"in_multi",
"ether_multi",
"mrt",
"",
"",
#if (NFSCLIENT || NFSSERVER)
"NFSV3 srvdesc",
"NFSV3 diroff",
"NFSV3 bigfh",
#else
"",
"",
"",
#endif
"MSDOSFS mount",
"MSDOSFS fat",
"MSDOSFS node",
"ttys",
"exec",
"miscfs mount",
"miscfs node",
"adosfs mount",
"adosfs node",
"adosfs anode",
"buf hdrs",
"ofile tabl",
"mbuf clust",
"",
"",
"",
"",
"",
"temp",
"key mgmt",
"DEVFS",
"IpFw/IpAcct",
"UDF node",
"UDF mount",
#if INET6
"IPv6 NDP",
"IPv6 options",
"IPv6 Misc",
#else
"",
"",
"",
#endif
"TCP Segment Q",
"IGMP state",
"",
"",
"specinfo",
"kqueue",
"",
"cluster_read",
"cluster_write",
"iov64",
"fileglob",
"kauth",
"dummynet",
"",
"macpipelabel",
"mactemp",
"sbuf",
"extattr",
"select",
#if TRAFFIC_MGT
"traffic_mgt",
#else
"",
#endif
#if FS_COMPRESSION
"decmpfs_cnode",
#else
"",
#endif
"ipmfilter",
"ipmsource",
"in6mfilter",
"ip6mopts",
"ip6msource",
#if FLOW_DIVERT
"flow_divert_pcb",
"flow_divert_group",
#else
"",
"",
#endif
"ip6cga",
#if NECP
"necp",
"necp_session_policy",
"necp_socket_policy",
"necp_ip_policy",
#else
"",
"",
"",
"",
#endif
"fdvnodedata"
"fddirbuf",
"netagent",
""
};
#define KMZ_CREATEZONE_ACCT ((void *)-3)
#define KMZ_CREATEZONE ((void *)-2)
#define KMZ_LOOKUPZONE ((void *)-1)
#define KMZ_MALLOC ((void *)0)
#define KMZ_SHAREZONE ((void *)1)
struct kmzones {
size_t kz_elemsize;
void *kz_zalloczone;
boolean_t kz_noencrypt;
} kmzones[M_LAST] = {
#define SOS(sname) sizeof (struct sname)
#define SOX(sname) -1
{ -1, 0, FALSE },
{ MSIZE, KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(socket), KMZ_CREATEZONE, TRUE },
{ SOS(inpcb), KMZ_LOOKUPZONE, TRUE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ SOS(rusage), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ MAXPATHLEN, KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(ucred), KMZ_CREATEZONE, FALSE },
{ SOS(pgrp), KMZ_CREATEZONE, FALSE },
{ SOS(session), KMZ_CREATEZONE, FALSE },
{ SOS(user32_iovec), KMZ_LOOKUPZONE, FALSE },
{ SOS(mount), KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if (NFSCLIENT || NFSSERVER)
{ SOS(nfsreq), KMZ_CREATEZONE, FALSE },
{ SOS(nfsmount),KMZ_CREATEZONE, FALSE },
{ SOS(nfsnode), KMZ_CREATEZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ SOS(vnode), KMZ_CREATEZONE, TRUE },
{ SOS(namecache), KMZ_CREATEZONE, FALSE },
#if QUOTA
{ SOX(dquot), KMZ_LOOKUPZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(plimit), KMZ_CREATEZONE, TRUE },
{ SOS(sigacts), KMZ_CREATEZONE_ACCT, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(fileproc),KMZ_CREATEZONE_ACCT, TRUE },
{ SOS(filedesc),KMZ_CREATEZONE_ACCT, TRUE },
{ SOX(lockf), KMZ_CREATEZONE_ACCT, TRUE },
{ SOS(proc), KMZ_CREATEZONE, FALSE },
{ SOS(pstats), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ M_FFSNODE, KMZ_SHAREZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ M_FFSNODE, KMZ_SHAREZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if (NFSCLIENT || NFSSERVER)
{ SOX(nfsrv_sock),
KMZ_CREATEZONE_ACCT, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOX(nfsrvcache),
KMZ_CREATEZONE_ACCT, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ SOX(ip_moptions),
KMZ_LOOKUPZONE, FALSE },
{ SOX(in_multi),KMZ_LOOKUPZONE, FALSE },
{ SOX(ether_multi),
KMZ_LOOKUPZONE, FALSE },
{ SOX(mrt), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if (NFSCLIENT || NFSSERVER)
{ SOS(nfsrv_descript),
KMZ_CREATEZONE_ACCT, FALSE },
{ SOS(nfsdmap), KMZ_CREATEZONE, FALSE },
{ SOS(fhandle), KMZ_LOOKUPZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(tty), KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, TRUE },
{ (NDFILE * OFILESIZE),
KMZ_CREATEZONE_ACCT, FALSE },
{ MCLBYTES, KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(specinfo),KMZ_CREATEZONE, TRUE },
{ SOS(kqueue), KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(cl_readahead), KMZ_CREATEZONE, TRUE },
{ SOS(cl_writebehind),KMZ_CREATEZONE, TRUE },
{ SOS(user64_iovec), KMZ_LOOKUPZONE, FALSE },
{ SOS(fileglob), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if FS_COMPRESSION
{ SOS(decmpfs_cnode),KMZ_CREATEZONE , FALSE},
#else
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if FLOW_DIVERT
{ SOS(flow_divert_pcb), KMZ_CREATEZONE, TRUE },
{ SOS(flow_divert_group), KMZ_CREATEZONE, TRUE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if NECP
{ SOS(necp_session_policy), KMZ_CREATEZONE, TRUE },
{ SOS(necp_kernel_socket_policy), KMZ_CREATEZONE, TRUE },
{ SOS(necp_kernel_ip_output_policy), KMZ_CREATEZONE, TRUE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#undef SOS
#undef SOX
};
extern zone_t kalloc_zone(vm_size_t);
void
kmeminit(void)
{
struct kmzones *kmz;
if ((sizeof(kmzones)/sizeof(kmzones[0])) != (sizeof(memname)/sizeof(memname[0]))) {
panic("kmeminit: kmzones has %lu elements but memname has %lu\n",
(sizeof(kmzones)/sizeof(kmzones[0])), (sizeof(memname)/sizeof(memname[0])));
}
kmz = kmzones;
while (kmz < &kmzones[M_LAST]) {
if (kmz->kz_elemsize == (size_t)(-1))
;
else
if (kmz->kz_zalloczone == KMZ_CREATEZONE ||
kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT) {
kmz->kz_zalloczone = zinit(kmz->kz_elemsize,
1024 * 1024, PAGE_SIZE,
memname[kmz - kmzones]);
zone_change(kmz->kz_zalloczone, Z_CALLERACCT,
(kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT));
if (kmz->kz_noencrypt == TRUE)
zone_change(kmz->kz_zalloczone, Z_NOENCRYPT, TRUE);
}
else if (kmz->kz_zalloczone == KMZ_LOOKUPZONE)
kmz->kz_zalloczone = kalloc_zone(kmz->kz_elemsize);
kmz++;
}
kmz = kmzones;
while (kmz < &kmzones[M_LAST]) {
if (kmz->kz_elemsize == (size_t)(-1))
;
else
if (kmz->kz_zalloczone == KMZ_SHAREZONE) {
kmz->kz_zalloczone =
kmzones[kmz->kz_elemsize].kz_zalloczone;
kmz->kz_elemsize =
kmzones[kmz->kz_elemsize].kz_elemsize;
}
kmz++;
}
}
void *
_MALLOC_external(
size_t size,
int type,
int flags);
void *
_MALLOC_external(
size_t size,
int type,
int flags)
{
static vm_allocation_site_t site = { VM_KERN_MEMORY_KALLOC, VM_TAG_BT };
return (__MALLOC(size, type, flags, &site));
}
void *
__MALLOC(
size_t size,
int type,
int flags,
vm_allocation_site_t *site)
{
void *addr = NULL;
vm_size_t msize = size;
if (type >= M_LAST)
panic("_malloc TYPE");
if (size == 0)
return (NULL);
if (msize != size) {
panic("Requested size to __MALLOC is too large (%llx)!\n", (uint64_t)size);
}
if (flags & M_NOWAIT) {
addr = (void *)kalloc_canblock(&msize, FALSE, site);
} else {
addr = (void *)kalloc_canblock(&msize, TRUE, site);
if (addr == NULL) {
if (flags & M_NULL) {
return NULL;
}
panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
}
}
if (!addr)
return (0);
if (flags & M_ZERO)
bzero(addr, size);
return (addr);
}
void
_FREE(
void *addr,
int type)
{
if (type >= M_LAST)
panic("_free TYPE");
if (!addr)
return;
kfree_addr(addr);
}
void *
__REALLOC(
void *addr,
size_t size,
int type,
int flags,
vm_allocation_site_t *site)
{
void *newaddr;
size_t alloc;
if (addr == NULL)
return (__MALLOC(size, type, flags, site));
alloc = kalloc_size(addr);
if (kalloc_bucket_size(size) == alloc) {
if (flags & M_ZERO) {
if (alloc < size)
bzero((uintptr_t)addr + alloc, (size - alloc));
else
bzero((uintptr_t)addr + size, (alloc - size));
}
return addr;
}
if ((newaddr = __MALLOC(size, type, flags, site)) == NULL)
return (NULL);
bcopy(addr, newaddr, MIN(size, alloc));
_FREE(addr, type);
return (newaddr);
}
void *
_MALLOC_ZONE_external(
size_t size,
int type,
int flags);
void *
_MALLOC_ZONE_external(
size_t size,
int type,
int flags)
{
return (__MALLOC_ZONE(size, type, flags, NULL));
}
void *
__MALLOC_ZONE(
size_t size,
int type,
int flags,
vm_allocation_site_t *site)
{
struct kmzones *kmz;
void *elem;
if (type >= M_LAST)
panic("_malloc_zone TYPE");
kmz = &kmzones[type];
if (kmz->kz_zalloczone == KMZ_MALLOC)
panic("_malloc_zone ZONE: type = %d", type);
if (kmz->kz_elemsize == (size_t)(-1))
panic("_malloc_zone XXX");
if (size == kmz->kz_elemsize)
if (flags & M_NOWAIT) {
elem = (void *)zalloc_noblock(kmz->kz_zalloczone);
} else {
elem = (void *)zalloc(kmz->kz_zalloczone);
}
else {
vm_size_t kalloc_size = size;
if (size > kalloc_size) {
elem = NULL;
} else if (flags & M_NOWAIT) {
elem = (void *)kalloc_canblock(&kalloc_size, FALSE, site);
} else {
elem = (void *)kalloc_canblock(&kalloc_size, TRUE, site);
}
}
if (elem && (flags & M_ZERO))
bzero(elem, size);
return (elem);
}
void
_FREE_ZONE(
void *elem,
size_t size,
int type)
{
struct kmzones *kmz;
if (type >= M_LAST)
panic("FREE_SIZE");
kmz = &kmzones[type];
if (kmz->kz_zalloczone == KMZ_MALLOC)
panic("free_zone ZONE");
if (kmz->kz_elemsize == (size_t)(-1))
panic("FREE_SIZE XXX");
if (size == kmz->kz_elemsize)
zfree(kmz->kz_zalloczone, elem);
else
kfree(elem, size);
}
#if CONFIG_ZLEAKS
SYSCTL_DECL(_kern_zleak);
SYSCTL_NODE(_kern, OID_AUTO, zleak, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "zleak");
static int
sysctl_zleak_active SYSCTL_HANDLER_ARGS
{
#pragma unused(arg1, arg2)
int oldval, val, error;
val = oldval = get_zleak_state();
error = sysctl_handle_int(oidp, &val, 0, req);
if (error || !req->newptr)
return (error);
if (val == 1 && oldval == 0) {
kern_return_t kr = zleak_activate();
if (KERN_SUCCESS != kr)
printf("zleak_active: failed to activate "
"live zone leak debugging (%d).\n", kr);
} if (val == 0 && oldval == 1) {
printf("zleak_active: active, cannot be disabled.\n");
return (EINVAL);
}
return (0);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, active,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, sysctl_zleak_active, "I", "zleak activity");
static int
sysctl_zleak_max_zonemap_size SYSCTL_HANDLER_ARGS
{
uint64_t zmap_max_size = *(vm_size_t *)arg1;
return sysctl_handle_quad(oidp, &zmap_max_size, arg2, req);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, max_zonemap_size,
CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED,
&zleak_max_zonemap_size, 0,
sysctl_zleak_max_zonemap_size, "Q", "zleak max zonemap size");
static int
sysctl_zleak_threshold SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg2)
int error;
uint64_t value = *(vm_size_t *)arg1;
error = sysctl_io_number(req, value, sizeof (value), &value, NULL);
if (error || !req->newptr)
return (error);
if (value > (uint64_t)zleak_max_zonemap_size)
return (ERANGE);
*(vm_size_t *)arg1 = value;
return (0);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, global_threshold,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
&zleak_global_tracking_threshold, 0,
sysctl_zleak_threshold, "Q", "zleak global threshold");
SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
&zleak_per_zone_tracking_threshold, 0,
sysctl_zleak_threshold, "Q", "zleak per-zone threshold");
#endif