#include <sys/param.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_pcb.h>
#include <sys/event.h>
#include <sys/eventvar.h>
#include <sys/proc_internal.h>
#include <sys/mount_internal.h>
#include <sys/vnode_internal.h>
#include <sys/ubc_internal.h>
#include <sys/namei.h>
#include <sys/file_internal.h>
#include <sys/filedesc.h>
#include <sys/tty.h>
#include <sys/quota.h>
#include <sys/uio_internal.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <hfs/hfs_cnode.h>
#include <miscfs/specfs/specdev.h>
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
#include <nfs/nfsnode.h>
#include <nfs/nfsmount.h>
#include <vfs/vfs_journal.h>
#include <mach/mach_types.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
void kmeminit(void) __attribute__((section("__TEXT, initcode")));
const char *memname[] = {
"free",
"mbuf",
"devbuf",
"socket",
"pcb",
"routetbl",
"hosttbl",
"fragtbl",
"zombie",
"ifaddr",
"soopts",
"soname",
"namei",
"gprof",
"ioctlops",
"mapmem",
"cred",
"pgrp",
"session",
"iov32",
"mount",
"fhandle",
#if (NFSCLIENT || NFSSERVER)
"NFS req",
"NFS mount",
"NFS node",
#else
"",
"",
"",
#endif
"vnodes",
"namecache",
#if QUOTA
"UFS quota",
#else
"",
#endif
"",
#if (SYSV_SEM || SYSV_MSG || SYSV_SHM)
"shm",
#else
"",
#endif
"plimit",
"sigacts",
"VM object",
"VM objhash",
"VM pmap",
"VM pvmap",
"VM pager",
"VM pgdata",
"fileproc",
"file desc",
"lockf",
"proc",
"pstats",
"LFS segment",
"LFS node",
"",
"MFS node",
"NQNFS Lease",
"NQNFS Host",
"Export Host",
#if (NFSCLIENT || NFSSERVER)
"NFS srvsock",
"NFS uid",
"NFS daemon",
#else
"",
"",
"",
#endif
"ip_moptions",
"in_multi",
"ether_multi",
"mrt",
"",
"",
#if (NFSCLIENT || NFSSERVER)
"NFSV3 srvdesc",
"NFSV3 diroff",
"NFSV3 bigfh",
#else
"",
"",
"",
#endif
"MSDOSFS mount",
"MSDOSFS fat",
"MSDOSFS node",
"ttys",
"exec",
"miscfs mount",
"miscfs node",
"adosfs mount",
"adosfs node",
"adosfs anode",
"buf hdrs",
"ofile tabl",
"mbuf clust",
#if HFS
"HFS mount",
"HFS node",
"HFS fork",
#else
"",
"",
"",
#endif
"ZFS mount",
"ZFS node",
"temp",
"key mgmt",
"DEVFS",
"IpFw/IpAcct",
"UDF node",
"UDF mount",
#if INET6
"IPv6 NDP",
"IPv6 options",
"IPv6 Misc",
#else
"",
"",
"",
#endif
"TCP Segment Q",
"IGMP state",
#if JOURNALING
"Journal",
"Transaction",
#else
"",
"",
#endif
"specinfo",
"kqueue",
#if HFS
"HFS dirhint",
#else
"",
#endif
"cluster_read",
"cluster_write",
"iov64",
"fileglob",
"kauth",
"dummynet",
#ifndef __LP64__
"unsafe_fsnode",
#else
"",
#endif
"macpipelabel",
"mactemp",
"sbuf",
"extattr",
"lctx",
#if TRAFFIC_MGT
"traffic_mgt",
#else
"",
#endif
#if HFS_COMPRESSION
"decmpfs_cnode",
#else
"",
#endif
"ipmfilter",
"ipmsource",
"in6mfilter",
"ip6mopts",
"ip6msource",
};
#define KMZ_CREATEZONE_ACCT ((void *)-3)
#define KMZ_CREATEZONE ((void *)-2)
#define KMZ_LOOKUPZONE ((void *)-1)
#define KMZ_MALLOC ((void *)0)
#define KMZ_SHAREZONE ((void *)1)
struct kmzones {
size_t kz_elemsize;
void *kz_zalloczone;
boolean_t kz_noencrypt;
} kmzones[M_LAST] = {
#define SOS(sname) sizeof (struct sname)
#define SOX(sname) -1
{ -1, 0, FALSE },
{ MSIZE, KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(socket), KMZ_CREATEZONE, TRUE },
{ SOS(inpcb), KMZ_LOOKUPZONE, TRUE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ SOS(rusage), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ M_MBUF, KMZ_SHAREZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ MAXPATHLEN, KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(ucred), KMZ_CREATEZONE, FALSE },
{ SOS(pgrp), KMZ_CREATEZONE, FALSE },
{ SOS(session), KMZ_CREATEZONE, FALSE },
{ SOS(user32_iovec), KMZ_LOOKUPZONE, FALSE },
{ SOS(mount), KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if (NFSCLIENT || NFSSERVER)
{ SOS(nfsreq), KMZ_CREATEZONE, FALSE },
{ SOS(nfsmount),KMZ_CREATEZONE, FALSE },
{ SOS(nfsnode), KMZ_CREATEZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ SOS(vnode), KMZ_CREATEZONE, TRUE },
{ SOS(namecache), KMZ_CREATEZONE, FALSE },
#if QUOTA
{ SOX(dquot), KMZ_LOOKUPZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(plimit), KMZ_CREATEZONE, TRUE },
{ SOS(sigacts), KMZ_CREATEZONE_ACCT, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(fileproc),KMZ_CREATEZONE_ACCT, TRUE },
{ SOS(filedesc),KMZ_CREATEZONE_ACCT, TRUE },
{ SOX(lockf), KMZ_CREATEZONE_ACCT, TRUE },
{ SOS(proc), KMZ_CREATEZONE, FALSE },
{ SOS(pstats), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ M_FFSNODE, KMZ_SHAREZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ M_FFSNODE, KMZ_SHAREZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if (NFSCLIENT || NFSSERVER)
{ SOX(nfsrv_sock),
KMZ_CREATEZONE_ACCT, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOX(nfsrvcache),
KMZ_CREATEZONE_ACCT, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ SOX(ip_moptions),
KMZ_LOOKUPZONE, FALSE },
{ SOX(in_multi),KMZ_LOOKUPZONE, FALSE },
{ SOX(ether_multi),
KMZ_LOOKUPZONE, FALSE },
{ SOX(mrt), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if (NFSCLIENT || NFSSERVER)
{ SOS(nfsrv_descript),
KMZ_CREATEZONE_ACCT, FALSE },
{ SOS(nfsdmap), KMZ_CREATEZONE, FALSE },
{ SOS(fhandle), KMZ_LOOKUPZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ SOS(tty), KMZ_CREATEZONE, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, TRUE },
{ (NDFILE * OFILESIZE),
KMZ_CREATEZONE_ACCT, FALSE },
{ MCLBYTES, KMZ_CREATEZONE, FALSE },
#if HFS
{ SOX(hfsmount),KMZ_LOOKUPZONE, FALSE },
{ SOS(cnode), KMZ_CREATEZONE, TRUE },
{ SOS(filefork),KMZ_CREATEZONE, TRUE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if JOURNALING
{ SOS(journal), KMZ_CREATEZONE, FALSE },
{ SOS(transaction), KMZ_CREATEZONE, FALSE },
#else
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#endif
{ SOS(specinfo),KMZ_CREATEZONE, TRUE },
{ SOS(kqueue), KMZ_CREATEZONE, FALSE },
#if HFS
{ SOS(directoryhint), KMZ_CREATEZONE, TRUE },
#else
{ 0, KMZ_MALLOC, FALSE },
#endif
{ SOS(cl_readahead), KMZ_CREATEZONE, TRUE },
{ SOS(cl_writebehind),KMZ_CREATEZONE, TRUE },
{ SOS(user64_iovec), KMZ_LOOKUPZONE, FALSE },
{ SOS(fileglob), KMZ_CREATEZONE, TRUE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#ifndef __LP64__
{ SOS(unsafe_fsnode),KMZ_CREATEZONE, TRUE },
#else
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#if HFS_COMPRESSION
{ SOS(decmpfs_cnode),KMZ_CREATEZONE , FALSE},
#else
{ 0, KMZ_MALLOC, FALSE },
#endif
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
{ 0, KMZ_MALLOC, FALSE },
#undef SOS
#undef SOX
};
extern zone_t kalloc_zone(vm_size_t);
void
kmeminit(void)
{
struct kmzones *kmz;
if ((sizeof(kmzones)/sizeof(kmzones[0])) != (sizeof(memname)/sizeof(memname[0]))) {
panic("kmeminit: kmzones has %lu elements but memname has %lu\n",
(sizeof(kmzones)/sizeof(kmzones[0])), (sizeof(memname)/sizeof(memname[0])));
}
kmz = kmzones;
while (kmz < &kmzones[M_LAST]) {
if (kmz->kz_elemsize == (size_t)(-1))
;
else
if (kmz->kz_zalloczone == KMZ_CREATEZONE ||
kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT) {
kmz->kz_zalloczone = zinit(kmz->kz_elemsize,
1024 * 1024, PAGE_SIZE,
memname[kmz - kmzones]);
zone_change(kmz->kz_zalloczone, Z_CALLERACCT,
(kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT));
if (kmz->kz_noencrypt == TRUE)
zone_change(kmz->kz_zalloczone, Z_NOENCRYPT, TRUE);
}
else if (kmz->kz_zalloczone == KMZ_LOOKUPZONE)
kmz->kz_zalloczone = kalloc_zone(kmz->kz_elemsize);
kmz++;
}
kmz = kmzones;
while (kmz < &kmzones[M_LAST]) {
if (kmz->kz_elemsize == (size_t)(-1))
;
else
if (kmz->kz_zalloczone == KMZ_SHAREZONE) {
kmz->kz_zalloczone =
kmzones[kmz->kz_elemsize].kz_zalloczone;
kmz->kz_elemsize =
kmzones[kmz->kz_elemsize].kz_elemsize;
}
kmz++;
}
}
struct _mhead {
size_t mlen;
char dat[0];
};
void *
_MALLOC(
size_t size,
int type,
int flags)
{
struct _mhead *hdr;
size_t memsize = sizeof (*hdr) + size;
if (type >= M_LAST)
panic("_malloc TYPE");
if (size == 0)
return (NULL);
if (flags & M_NOWAIT) {
hdr = (void *)kalloc_noblock(memsize);
} else {
hdr = (void *)kalloc(memsize);
if (hdr == NULL) {
panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
}
}
if (!hdr)
return (0);
hdr->mlen = memsize;
if (flags & M_ZERO)
bzero(hdr->dat, size);
return (hdr->dat);
}
void
_FREE(
void *addr,
int type)
{
struct _mhead *hdr;
if (type >= M_LAST)
panic("_free TYPE");
if (!addr)
return;
hdr = addr; hdr--;
kfree(hdr, hdr->mlen);
}
void *
_REALLOC(
void *addr,
size_t size,
int type,
int flags)
{
struct _mhead *hdr;
void *newaddr;
size_t alloc;
if (addr == NULL)
return (_MALLOC(size, type, flags));
if ((newaddr = _MALLOC(size, type, flags)) == NULL)
return (NULL);
hdr = addr;
--hdr;
alloc = hdr->mlen - sizeof (*hdr);
bcopy(addr, newaddr, MIN(size, alloc));
_FREE(addr, type);
return (newaddr);
}
void *
_MALLOC_ZONE(
size_t size,
int type,
int flags)
{
struct kmzones *kmz;
void *elem;
if (type >= M_LAST)
panic("_malloc_zone TYPE");
kmz = &kmzones[type];
if (kmz->kz_zalloczone == KMZ_MALLOC)
panic("_malloc_zone ZONE: type = %d", type);
if (kmz->kz_elemsize == (size_t)(-1))
panic("_malloc_zone XXX");
if (size == kmz->kz_elemsize)
if (flags & M_NOWAIT) {
elem = (void *)zalloc_noblock(kmz->kz_zalloczone);
} else {
elem = (void *)zalloc(kmz->kz_zalloczone);
}
else
if (flags & M_NOWAIT) {
elem = (void *)kalloc_noblock(size);
} else {
elem = (void *)kalloc(size);
}
return (elem);
}
void
_FREE_ZONE(
void *elem,
size_t size,
int type)
{
struct kmzones *kmz;
if (type >= M_LAST)
panic("FREE_SIZE");
kmz = &kmzones[type];
if (kmz->kz_zalloczone == KMZ_MALLOC)
panic("free_zone ZONE");
if (kmz->kz_elemsize == (size_t)(-1))
panic("FREE_SIZE XXX");
if (size == kmz->kz_elemsize)
zfree(kmz->kz_zalloczone, elem);
else
kfree(elem, size);
}
#if CONFIG_ZLEAKS
SYSCTL_DECL(_kern_zleak);
SYSCTL_NODE(_kern, OID_AUTO, zleak, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "zleak");
static int
sysctl_zleak_active SYSCTL_HANDLER_ARGS
{
#pragma unused(arg1, arg2)
int oldval, val, error;
val = oldval = get_zleak_state();
error = sysctl_handle_int(oidp, &val, 0, req);
if (error || !req->newptr)
return (error);
if (val == 1 && oldval == 0) {
kern_return_t kr = zleak_activate();
if (KERN_SUCCESS != kr)
printf("zleak_active: failed to activate "
"live zone leak debugging (%d).\n", kr);
} if (val == 0 && oldval == 1) {
printf("zleak_active: active, cannot be disabled.\n");
return (EINVAL);
}
return (0);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, active,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, sysctl_zleak_active, "I", "zleak activity");
static int
sysctl_zleak_max_zonemap_size SYSCTL_HANDLER_ARGS
{
uint64_t zmap_max_size = *(vm_size_t *)arg1;
return sysctl_handle_quad(oidp, &zmap_max_size, arg2, req);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, max_zonemap_size,
CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED,
&zleak_max_zonemap_size, 0,
sysctl_zleak_max_zonemap_size, "Q", "zleak max zonemap size");
static int
sysctl_zleak_threshold SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg2)
int error;
uint64_t value = *(vm_size_t *)arg1;
error = sysctl_io_number(req, value, sizeof (value), &value, NULL);
if (error || !req->newptr)
return (error);
if (value > (uint64_t)zleak_max_zonemap_size)
return (ERANGE);
*(vm_size_t *)arg1 = value;
return (0);
}
SYSCTL_PROC(_kern_zleak, OID_AUTO, global_threshold,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
&zleak_global_tracking_threshold, 0,
sysctl_zleak_threshold, "Q", "zleak global threshold");
SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
&zleak_per_zone_tracking_threshold, 0,
sysctl_zleak_threshold, "Q", "zleak per-zone threshold");
#endif