/* * Copyright (c) 1999, 2006-2008 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this * file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ */ #include #include "magmallocProvider.h" #import #import #import #import #import #import #import #import #import #import #import #import #import #import #import "scalable_malloc.h" #import "stack_logging.h" #import "malloc_printf.h" #import "_simple.h" /* * MALLOC_ABSOLUTE_MAX_SIZE - There are many instances of addition to a * user-specified size_t, which can cause overflow (and subsequent crashes) * for values near SIZE_T_MAX. Rather than add extra "if" checks everywhere * this occurs, it is easier to just set an absolute maximum request size, * and immediately return an error if the requested size exceeds this maximum. * Of course, values less than this absolute max can fail later if the value * is still too large for the available memory. The largest value added * seems to be PAGE_SIZE (in the macro round_page()), so to be safe, we set * the maximum to be 2 * PAGE_SIZE less than SIZE_T_MAX. */ #define MALLOC_ABSOLUTE_MAX_SIZE (SIZE_T_MAX - (2 * PAGE_SIZE)) #define USE_SLEEP_RATHER_THAN_ABORT 0 typedef void (malloc_logger_t)(uint32_t type, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t result, uint32_t num_hot_frames_to_skip); __private_extern__ pthread_lock_t _malloc_lock = 0; // initialized in __libc_init /* The following variables are exported for the benefit of performance tools * * It should always be safe to first read malloc_num_zones, then read * malloc_zones without taking the lock, if only iteration is required */ unsigned malloc_num_zones = 0; unsigned malloc_num_zones_allocated = 0; malloc_zone_t **malloc_zones = 0; malloc_logger_t *malloc_logger = NULL; unsigned malloc_debug_flags = 0; unsigned malloc_check_start = 0; // 0 means don't check unsigned malloc_check_counter = 0; unsigned malloc_check_each = 1000; /* global flag to suppress ASL logging e.g. for syslogd */ int _malloc_no_asl_log = 0; static int malloc_check_sleep = 100; // default 100 second sleep static int malloc_check_abort = 0; // default is to sleep, not abort static int malloc_debug_file = STDERR_FILENO; /* * State indicated by malloc_def_zone_state * 0 - the default zone has not yet been created * 1 - a Malloc* environment variable has been set * 2 - the default zone has been created and an environment variable scan done * 3 - a new default zone has been created and another environment variable scan */ __private_extern__ int malloc_def_zone_state = 0; __private_extern__ malloc_zone_t *__zone0 = NULL; static const char Malloc_Facility[] = "com.apple.Libsystem.malloc"; #define MALLOC_LOCK() LOCK(_malloc_lock) #define MALLOC_UNLOCK() UNLOCK(_malloc_lock) #define MALLOC_LOG_TYPE_ALLOCATE stack_logging_type_alloc #define MALLOC_LOG_TYPE_DEALLOCATE stack_logging_type_dealloc #define MALLOC_LOG_TYPE_HAS_ZONE stack_logging_flag_zone #define MALLOC_LOG_TYPE_CLEARED stack_logging_flag_cleared /********* Utilities ************/ static inline malloc_zone_t * find_registered_zone(const void *, size_t *) __attribute__((always_inline)); static inline malloc_zone_t * find_registered_zone(const void *ptr, size_t *returned_size) { // Returns a zone which contains ptr, else NULL unsigned index; malloc_zone_t **zones = malloc_zones; for (index = 0; index < malloc_num_zones; ++index, ++zones) { malloc_zone_t *zone = *zones; size_t size = zone->size(zone, ptr); if (size) { // Claimed by this zone? if (returned_size) *returned_size = size; return zone; } } // Unclaimed by any zone. if (returned_size) *returned_size = 0; return NULL; } __private_extern__ __attribute__((noinline)) void malloc_error_break(void) { // Provides a non-inlined place for various malloc error procedures to call // that will be called after an error message appears. It does not make // sense for developers to call this function, so it is marked // __private_extern__ to prevent it from becoming API. MAGMALLOC_MALLOCERRORBREAK(); // DTrace USDT probe } __private_extern__ boolean_t __stack_logging_locked(); __private_extern__ __attribute__((noinline)) int malloc_gdb_po_unsafe(void) { // In order to implement "po" other data formatters in gdb, the debugger // calls functions that call malloc. The debugger will only run one thread // of the program in this case, so if another thread is holding a zone lock, // gdb may deadlock in this case. // // Iterate over the zones in malloc_zones, and call "trylock" on the zone // lock. If trylock succeeds, unlock it, otherwise return "locked". Returns // 0 == safe, 1 == locked/unsafe. if (__stack_logging_locked()) return 1; malloc_zone_t **zones = malloc_zones; unsigned i, e = malloc_num_zones; for (i = 0; i != e; ++i) { malloc_zone_t *zone = zones[i]; // Version must be >= 5 to look at the new introspection field. if (zone->version < 5) continue; if (zone->introspect->zone_locked && zone->introspect->zone_locked(zone)) return 1; } return 0; } /********* Creation and destruction ************/ static void set_flags_from_environment(void); static void malloc_zone_register_while_locked(malloc_zone_t *zone) { size_t protect_size; unsigned i; /* scan the list of zones, to see if this zone is already registered. If * so, print an error message and return. */ for (i = 0; i != malloc_num_zones; ++i) if (zone == malloc_zones[i]) { _malloc_printf(ASL_LEVEL_ERR, "Attempted to register zone more than once: %p\n", zone); return; } if (malloc_num_zones == malloc_num_zones_allocated) { size_t malloc_zones_size = malloc_num_zones * sizeof(malloc_zone_t *); size_t alloc_size = malloc_zones_size + vm_page_size; malloc_zone_t **new_zones = mmap(0, alloc_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, VM_MAKE_TAG(VM_MEMORY_MALLOC), 0); /* If there were previously allocated malloc zones, we need to copy them * out of the previous array and into the new zones array */ if (malloc_zones) memcpy(new_zones, malloc_zones, malloc_zones_size); /* Update the malloc_zones pointer, which we leak if it was previously * allocated, and the number of zones allocated */ protect_size = alloc_size; malloc_zones = new_zones; malloc_num_zones_allocated = alloc_size / sizeof(malloc_zone_t *); } else { /* If we don't need to reallocate zones, we need to briefly change the * page protection the malloc zones to allow writes */ protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); } malloc_zones[malloc_num_zones++] = zone; /* Finally, now that the zone is registered, disallow write access to the * malloc_zones array */ vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); //_malloc_printf(ASL_LEVEL_INFO, "Registered malloc_zone %p in malloc_zones %p [%u zones, %u bytes]\n", zone, malloc_zones, malloc_num_zones, protect_size); } static void _malloc_initialize(void) { MALLOC_LOCK(); if (malloc_def_zone_state < 2) { unsigned n; malloc_zone_t *zone; malloc_def_zone_state += 2; set_flags_from_environment(); // will only set flags up to two times n = malloc_num_zones; zone = create_scalable_zone(0, malloc_debug_flags); malloc_zone_register_while_locked(zone); malloc_set_zone_name(zone, "DefaultMallocZone"); if (n != 0) { // make the default first, for efficiency unsigned protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); malloc_zone_t *hold = malloc_zones[0]; if(hold->zone_name && strcmp(hold->zone_name, "DefaultMallocZone") == 0) { free((void *)hold->zone_name); hold->zone_name = NULL; } vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); malloc_zones[0] = malloc_zones[n]; malloc_zones[n] = hold; vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); } // _malloc_printf(ASL_LEVEL_INFO, "%d registered zones\n", malloc_num_zones); // _malloc_printf(ASL_LEVEL_INFO, "malloc_zones is at %p; malloc_num_zones is at %p\n", (unsigned)&malloc_zones, (unsigned)&malloc_num_zones); } MALLOC_UNLOCK(); } static inline malloc_zone_t *inline_malloc_default_zone(void) __attribute__((always_inline)); static inline malloc_zone_t * inline_malloc_default_zone(void) { if (malloc_def_zone_state < 2) _malloc_initialize(); // _malloc_printf(ASL_LEVEL_INFO, "In inline_malloc_default_zone with %d %d\n", malloc_num_zones, malloc_has_debug_zone); return malloc_zones[0]; } malloc_zone_t * malloc_default_zone(void) { return inline_malloc_default_zone(); } malloc_zone_t * malloc_default_purgeable_zone(void) { static malloc_zone_t *dpz; if (!dpz) { malloc_zone_t *tmp = create_purgeable_zone(0, malloc_default_zone(), malloc_debug_flags); malloc_zone_register(tmp); malloc_set_zone_name(tmp, "DefaultPurgeableMallocZone"); if (!__sync_bool_compare_and_swap(&dpz, NULL, tmp)) malloc_destroy_zone(tmp); } return dpz; } // For debugging, allow stack logging to both memory and disk to compare their results. static void stack_logging_log_stack_debug(uint32_t type_flags, uintptr_t zone_ptr, uintptr_t size, uintptr_t ptr_arg, uintptr_t return_val, uint32_t num_hot_to_skip) { __disk_stack_logging_log_stack(type_flags, zone_ptr, size, ptr_arg, return_val, num_hot_to_skip); stack_logging_log_stack(type_flags, zone_ptr, size, ptr_arg, return_val, num_hot_to_skip); } static void set_flags_from_environment(void) { const char *flag; int fd; char **env = * _NSGetEnviron(); char **p; char *c; if (malloc_debug_file != STDERR_FILENO) { close(malloc_debug_file); malloc_debug_file = STDERR_FILENO; } #if __LP64__ malloc_debug_flags = SCALABLE_MALLOC_ABORT_ON_CORRUPTION; // Set always on 64-bit processes #else malloc_debug_flags = 0; #endif stack_logging_enable_logging = 0; stack_logging_dontcompact = 0; malloc_logger = NULL; malloc_check_start = 0; malloc_check_each = 1000; malloc_check_abort = 0; malloc_check_sleep = 100; /* * Given that all environment variables start with "Malloc" we optimize by scanning quickly * first the environment, therefore avoiding repeated calls to getenv(). * If we are setu/gid these flags are ignored to prevent a malicious invoker from changing * our behaviour. */ for (p = env; (c = *p) != NULL; ++p) { if (!strncmp(c, "Malloc", 6)) { if (issetugid()) return; break; } } if (c == NULL) return; flag = getenv("MallocLogFile"); if (flag) { fd = open(flag, O_WRONLY|O_APPEND|O_CREAT, 0644); if (fd >= 0) { malloc_debug_file = fd; fcntl(fd, F_SETFD, 0); // clear close-on-exec flag XXX why? } else { malloc_printf("Could not open %s, using stderr\n", flag); } } if (getenv("MallocGuardEdges")) { malloc_debug_flags = SCALABLE_MALLOC_ADD_GUARD_PAGES; _malloc_printf(ASL_LEVEL_INFO, "protecting edges\n"); if (getenv("MallocDoNotProtectPrelude")) { malloc_debug_flags |= SCALABLE_MALLOC_DONT_PROTECT_PRELUDE; _malloc_printf(ASL_LEVEL_INFO, "... but not protecting prelude guard page\n"); } if (getenv("MallocDoNotProtectPostlude")) { malloc_debug_flags |= SCALABLE_MALLOC_DONT_PROTECT_POSTLUDE; _malloc_printf(ASL_LEVEL_INFO, "... but not protecting postlude guard page\n"); } } flag = getenv("MallocStackLogging"); if (!flag) { flag = getenv("MallocStackLoggingNoCompact"); stack_logging_dontcompact = 1; } // For debugging, the MallocStackLogging or MallocStackLoggingNoCompact environment variables can be set to // values of "memory", "disk", or "both" to control which stack logging mechanism to use. Those strings appear // in the flag variable, and the strtoul() call below will return 0, so then we can do string comparison on the // value of flag. The default stack logging now is disk stack logging, since memory stack logging is not 64-bit-aware. if (flag) { unsigned long val = strtoul(flag, NULL, 0); if (val == 1) val = 0; if (val == -1) val = 0; if (val) { malloc_logger = (void *)val; _malloc_printf(ASL_LEVEL_INFO, "recording stacks using recorder %p\n", malloc_logger); } else if (strcmp(flag,"memory") == 0) { malloc_logger = (malloc_logger_t *)stack_logging_log_stack; _malloc_printf(ASL_LEVEL_INFO, "recording malloc stacks in memory using standard recorder\n"); } else if (strcmp(flag,"both") == 0) { malloc_logger = stack_logging_log_stack_debug; _malloc_printf(ASL_LEVEL_INFO, "recording malloc stacks to both memory and disk for comparison debugging\n"); } else { // the default is to log to disk malloc_logger = __disk_stack_logging_log_stack; _malloc_printf(ASL_LEVEL_INFO, "recording malloc stacks to disk using standard recorder\n"); } stack_logging_enable_logging = 1; if (stack_logging_dontcompact) { if (malloc_logger == __disk_stack_logging_log_stack) { _malloc_printf(ASL_LEVEL_INFO, "stack logging compaction turned off; size of log files on disk can increase rapidly\n"); } else { _malloc_printf(ASL_LEVEL_INFO, "stack logging compaction turned off; VM can increase rapidly\n"); } } } if (getenv("MallocScribble")) { malloc_debug_flags |= SCALABLE_MALLOC_DO_SCRIBBLE; _malloc_printf(ASL_LEVEL_INFO, "enabling scribbling to detect mods to free blocks\n"); } if (getenv("MallocErrorAbort")) { malloc_debug_flags |= SCALABLE_MALLOC_ABORT_ON_ERROR; _malloc_printf(ASL_LEVEL_INFO, "enabling abort() on bad malloc or free\n"); } #if __LP64__ /* initialization above forces SCALABLE_MALLOC_ABORT_ON_CORRUPTION of 64-bit processes */ #else if (getenv("MallocCorruptionAbort")) { // Set from an environment variable in 32-bit processes malloc_debug_flags |= SCALABLE_MALLOC_ABORT_ON_CORRUPTION; } #endif flag = getenv("MallocCheckHeapStart"); if (flag) { malloc_check_start = strtoul(flag, NULL, 0); if (malloc_check_start == 0) malloc_check_start = 1; if (malloc_check_start == -1) malloc_check_start = 1; flag = getenv("MallocCheckHeapEach"); if (flag) { malloc_check_each = strtoul(flag, NULL, 0); if (malloc_check_each == 0) malloc_check_each = 1; if (malloc_check_each == -1) malloc_check_each = 1; } _malloc_printf(ASL_LEVEL_INFO, "checks heap after %dth operation and each %d operations\n", malloc_check_start, malloc_check_each); flag = getenv("MallocCheckHeapAbort"); if (flag) malloc_check_abort = strtol(flag, NULL, 0); if (malloc_check_abort) _malloc_printf(ASL_LEVEL_INFO, "will abort on heap corruption\n"); else { flag = getenv("MallocCheckHeapSleep"); if (flag) malloc_check_sleep = strtol(flag, NULL, 0); if (malloc_check_sleep > 0) _malloc_printf(ASL_LEVEL_INFO, "will sleep for %d seconds on heap corruption\n", malloc_check_sleep); else if (malloc_check_sleep < 0) _malloc_printf(ASL_LEVEL_INFO, "will sleep once for %d seconds on heap corruption\n", -malloc_check_sleep); else _malloc_printf(ASL_LEVEL_INFO, "no sleep on heap corruption\n"); } } if (getenv("MallocHelp")) { _malloc_printf(ASL_LEVEL_INFO, "environment variables that can be set for debug:\n" "- MallocLogFile to create/append messages to file instead of stderr\n" "- MallocGuardEdges to add 2 guard pages for each large block\n" "- MallocDoNotProtectPrelude to disable protection (when previous flag set)\n" "- MallocDoNotProtectPostlude to disable protection (when previous flag set)\n" "- MallocStackLogging to record all stacks. Tools like leaks can then be applied\n" "- MallocStackLoggingNoCompact to record all stacks. Needed for malloc_history\n" "- MallocStackLoggingDirectory to set location of stack logs, which can grow large; default is /tmp\n" "- MallocScribble to detect writing on free blocks and missing initializers:\n" " 0x55 is written upon free and 0xaa is written on allocation\n" "- MallocCheckHeapStart to start checking the heap after operations\n" "- MallocCheckHeapEach to repeat the checking of the heap after operations\n" "- MallocCheckHeapSleep to sleep seconds on heap corruption\n" "- MallocCheckHeapAbort to abort on heap corruption if is non-zero\n" "- MallocCorruptionAbort to abort on malloc errors, but not on out of memory for 32-bit processes\n" " MallocCorruptionAbort is always set on 64-bit processes\n" "- MallocErrorAbort to abort on any malloc error, including out of memory\n" "- MallocHelp - this help!\n"); } } malloc_zone_t * malloc_create_zone(vm_size_t start_size, unsigned flags) { malloc_zone_t *zone; /* start_size doesn't seemed to actually be used, but we test anyways */ if (start_size > MALLOC_ABSOLUTE_MAX_SIZE) { return NULL; } if (malloc_def_zone_state < 2) _malloc_initialize(); zone = create_scalable_zone(start_size, flags | malloc_debug_flags); malloc_zone_register(zone); return zone; } /* * For use by CheckFix: establish a new default zone whose behavior is, apart from * the use of death-row and per-CPU magazines, that of Leopard. */ void malloc_create_legacy_default_zone(void) { malloc_zone_t *zone; int i; if (malloc_def_zone_state < 2) _malloc_initialize(); zone = create_legacy_scalable_zone(0, malloc_debug_flags); MALLOC_LOCK(); malloc_zone_register_while_locked(zone); // // Establish the legacy scalable zone just created as the default zone. // malloc_zone_t *hold = malloc_zones[0]; if(hold->zone_name && strcmp(hold->zone_name, "DefaultMallocZone") == 0) { free((void *)hold->zone_name); hold->zone_name = NULL; } malloc_set_zone_name(zone, "DefaultMallocZone"); unsigned protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); // assert(zone == malloc_zones[malloc_num_zones - 1]; for (i = malloc_num_zones - 1; i > 0; --i) { malloc_zones[i] = malloc_zones[i - 1]; } malloc_zones[0] = zone; vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); MALLOC_UNLOCK(); } void malloc_destroy_zone(malloc_zone_t *zone) { malloc_zone_unregister(zone); zone->destroy(zone); } /* called from the {put,set,unset}env routine */ __private_extern__ void __malloc_check_env_name(const char *name) { MALLOC_LOCK(); if(malloc_def_zone_state == 2 && strncmp(name, "Malloc", 6) == 0) malloc_def_zone_state = 1; MALLOC_UNLOCK(); } /********* Block creation and manipulation ************/ extern const char *__crashreporter_info__; static void internal_check(void) { static vm_address_t *frames = NULL; static unsigned num_frames; if (malloc_zone_check(NULL)) { if (!frames) vm_allocate(mach_task_self(), (void *)&frames, vm_page_size, 1); thread_stack_pcs(frames, vm_page_size/sizeof(vm_address_t) - 1, &num_frames); } else { _SIMPLE_STRING b = _simple_salloc(); if (b) _simple_sprintf(b, "*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1); else _malloc_printf(MALLOC_PRINTF_NOLOG, "*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1); malloc_printf("*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1); if (frames) { unsigned index = 1; if (b) { _simple_sappend(b, "Stack for last operation where the malloc check succeeded: "); while (index < num_frames) _simple_sprintf(b, "%p ", frames[index++]); malloc_printf("%s\n(Use 'atos' for a symbolic stack)\n", _simple_string(b)); } else { /* * Should only get here if vm_allocate() can't get a single page of * memory, implying _simple_asl_log() would also fail. So we just * print to the file descriptor. */ _malloc_printf(MALLOC_PRINTF_NOLOG, "Stack for last operation where the malloc check succeeded: "); while (index < num_frames) _malloc_printf(MALLOC_PRINTF_NOLOG, "%p ", frames[index++]); _malloc_printf(MALLOC_PRINTF_NOLOG, "\n(Use 'atos' for a symbolic stack)\n"); } } if (malloc_check_each > 1) { unsigned recomm_each = (malloc_check_each > 10) ? malloc_check_each/10 : 1; unsigned recomm_start = (malloc_check_counter > malloc_check_each+1) ? malloc_check_counter-1-malloc_check_each : 1; malloc_printf("*** Recommend using 'setenv MallocCheckHeapStart %d; setenv MallocCheckHeapEach %d' to narrow down failure\n", recomm_start, recomm_each); } if (malloc_check_abort) { __crashreporter_info__ = b ? _simple_string(b) : "*** MallocCheckHeap: FAILED check"; abort(); } else if (b) _simple_sfree(b); if (malloc_check_sleep > 0) { _malloc_printf(ASL_LEVEL_NOTICE, "*** Sleeping for %d seconds to leave time to attach\n", malloc_check_sleep); sleep(malloc_check_sleep); } else if (malloc_check_sleep < 0) { _malloc_printf(ASL_LEVEL_NOTICE, "*** Sleeping once for %d seconds to leave time to attach\n", -malloc_check_sleep); sleep(-malloc_check_sleep); malloc_check_sleep = 0; } } malloc_check_start += malloc_check_each; } void * malloc_zone_malloc(malloc_zone_t *zone, size_t size) { void *ptr; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } if (size > MALLOC_ABSOLUTE_MAX_SIZE) { return NULL; } ptr = zone->malloc(zone, size); if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)ptr, 0); return ptr; } void * malloc_zone_calloc(malloc_zone_t *zone, size_t num_items, size_t size) { void *ptr; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } if (size > MALLOC_ABSOLUTE_MAX_SIZE) { return NULL; } ptr = zone->calloc(zone, num_items, size); if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE | MALLOC_LOG_TYPE_CLEARED, (uintptr_t)zone, (uintptr_t)(num_items * size), 0, (uintptr_t)ptr, 0); return ptr; } void * malloc_zone_valloc(malloc_zone_t *zone, size_t size) { void *ptr; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } if (size > MALLOC_ABSOLUTE_MAX_SIZE) { return NULL; } ptr = zone->valloc(zone, size); if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)ptr, 0); return ptr; } void * malloc_zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) { void *new_ptr; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } if (size > MALLOC_ABSOLUTE_MAX_SIZE) { return NULL; } new_ptr = zone->realloc(zone, ptr, size); if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)ptr, (uintptr_t)size, (uintptr_t)new_ptr, 0); return new_ptr; } void malloc_zone_free(malloc_zone_t *zone, void *ptr) { if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)ptr, 0, 0, 0); if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } zone->free(zone, ptr); } static void malloc_zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) { if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)ptr, 0, 0, 0); if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } zone->free_definite_size(zone, ptr, size); } malloc_zone_t * malloc_zone_from_ptr(const void *ptr) { if (!ptr) return NULL; else return find_registered_zone(ptr, NULL); } void * malloc_zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) { void *ptr; if (zone->version < 5) // Version must be >= 5 to look at the new memalign field. return NULL; if (!(zone->memalign)) return NULL; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } if (size > MALLOC_ABSOLUTE_MAX_SIZE) { return NULL; } if (alignment < sizeof( void *) || // excludes 0 == alignment 0 != (alignment & (alignment - 1))) { // relies on sizeof(void *) being a power of two. return NULL; } ptr = zone->memalign(zone, alignment, size); if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)ptr, 0); return ptr; } /********* Functions for zone implementors ************/ void malloc_zone_register(malloc_zone_t *zone) { MALLOC_LOCK(); malloc_zone_register_while_locked(zone); MALLOC_UNLOCK(); } void malloc_zone_unregister(malloc_zone_t *z) { unsigned index; if (malloc_num_zones == 0) return; MALLOC_LOCK(); for (index = 0; index < malloc_num_zones; ++index) { if (z != malloc_zones[index]) continue; // Modify the page to be allow write access, so that we can update the // malloc_zones array. size_t protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); // If we found a match, swap it with the entry on the back of the list // and null out the back of the list. malloc_zones[index] = malloc_zones[malloc_num_zones - 1]; malloc_zones[malloc_num_zones - 1] = NULL; --malloc_num_zones; vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); MALLOC_UNLOCK(); return; } MALLOC_UNLOCK(); malloc_printf("*** malloc_zone_unregister() failed for %p\n", z); } void malloc_set_zone_name(malloc_zone_t *z, const char *name) { char *newName; if (z->zone_name) { free((char *)z->zone_name); z->zone_name = NULL; } newName = malloc_zone_malloc(z, strlen(name) + 1); strcpy(newName, name); z->zone_name = (const char *)newName; } const char * malloc_get_zone_name(malloc_zone_t *zone) { return zone->zone_name; } /* * XXX malloc_printf now uses _simple_*printf. It only deals with a * subset of printf format specifiers, but it doesn't call malloc. */ __private_extern__ void _malloc_vprintf(int flags, const char *format, va_list ap) { _SIMPLE_STRING b; if (_malloc_no_asl_log || (flags & MALLOC_PRINTF_NOLOG) || (b = _simple_salloc()) == NULL) { if (!(flags & MALLOC_PRINTF_NOPREFIX)) { if (__is_threaded) { /* XXX somewhat rude 'knowing' that pthread_t is a pointer */ _simple_dprintf(malloc_debug_file, "%s(%d,%p) malloc: ", getprogname(), getpid(), (void *)pthread_self()); } else { _simple_dprintf(malloc_debug_file, "%s(%d) malloc: ", getprogname(), getpid()); } } _simple_vdprintf(malloc_debug_file, format, ap); return; } if (!(flags & MALLOC_PRINTF_NOPREFIX)) { if (__is_threaded) { /* XXX somewhat rude 'knowing' that pthread_t is a pointer */ _simple_sprintf(b, "%s(%d,%p) malloc: ", getprogname(), getpid(), (void *)pthread_self()); } else { _simple_sprintf(b, "%s(%d) malloc: ", getprogname(), getpid()); } } _simple_vsprintf(b, format, ap); _simple_put(b, malloc_debug_file); _simple_asl_log(flags & MALLOC_PRINTF_LEVEL_MASK, Malloc_Facility, _simple_string(b)); _simple_sfree(b); } __private_extern__ void _malloc_printf(int flags, const char *format, ...) { va_list ap; va_start(ap, format); _malloc_vprintf(flags, format, ap); va_end(ap); } void malloc_printf(const char *format, ...) { va_list ap; va_start(ap, format); _malloc_vprintf(ASL_LEVEL_ERR, format, ap); va_end(ap); } /********* Generic ANSI callouts ************/ void * malloc(size_t size) { void *retval; retval = malloc_zone_malloc(inline_malloc_default_zone(), size); if (retval == NULL) { errno = ENOMEM; } return retval; } void * calloc(size_t num_items, size_t size) { void *retval; retval = malloc_zone_calloc(inline_malloc_default_zone(), num_items, size); if (retval == NULL) { errno = ENOMEM; } return retval; } void free(void *ptr) { malloc_zone_t *zone; size_t size; if (!ptr) return; zone = find_registered_zone(ptr, &size); if (!zone) { malloc_printf("*** error for object %p: pointer being freed was not allocated\n" "*** set a breakpoint in malloc_error_break to debug\n", ptr); malloc_error_break(); if ((malloc_debug_flags & (SCALABLE_MALLOC_ABORT_ON_CORRUPTION|SCALABLE_MALLOC_ABORT_ON_ERROR))) abort(); } else if (zone->version >= 6 && zone->free_definite_size) malloc_zone_free_definite_size(zone, ptr, size); else malloc_zone_free(zone, ptr); } void * realloc(void *in_ptr, size_t new_size) { void *retval; void *old_ptr; malloc_zone_t *zone; size_t old_size = 0; // SUSv3: "If size is 0 and ptr is not a null pointer, the object // pointed to is freed. If the space cannot be allocated, the object // shall remain unchanged." Also "If size is 0, either a null pointer // or a unique pointer that can be successfully passed to free() shall // be returned." We choose to allocate a minimum size object by calling // malloc_zone_malloc with zero size, which matches "If ptr is a null // pointer, realloc() shall be equivalent to malloc() for the specified // size." So we only free the original memory if the allocation succeeds. old_ptr = (new_size == 0) ? NULL : in_ptr; if (!old_ptr) { retval = malloc_zone_malloc(inline_malloc_default_zone(), new_size); } else { zone = find_registered_zone(old_ptr, &old_size); if (zone && old_size >= new_size) return old_ptr; if (!zone) zone = inline_malloc_default_zone(); retval = malloc_zone_realloc(zone, old_ptr, new_size); } if (retval == NULL) { errno = ENOMEM; } else if (new_size == 0) { free(in_ptr); } return retval; } void * valloc(size_t size) { void *retval; malloc_zone_t *zone = inline_malloc_default_zone(); retval = malloc_zone_valloc(zone, size); if (retval == NULL) { errno = ENOMEM; } return retval; } extern void vfree(void *ptr) { free(ptr); } size_t malloc_size(const void *ptr) { size_t size = 0; if (!ptr) return size; (void)find_registered_zone(ptr, &size); return size; } size_t malloc_good_size (size_t size) { malloc_zone_t *zone = inline_malloc_default_zone(); return zone->introspect->good_size(zone, size); } /* * The posix_memalign() function shall allocate size bytes aligned on a boundary specified by alignment, * and shall return a pointer to the allocated memory in memptr. * The value of alignment shall be a multiple of sizeof( void *), that is also a power of two. * Upon successful completion, the value pointed to by memptr shall be a multiple of alignment. * * Upon successful completion, posix_memalign() shall return zero; otherwise, * an error number shall be returned to indicate the error. * * The posix_memalign() function shall fail if: * EINVAL * The value of the alignment parameter is not a power of two multiple of sizeof( void *). * ENOMEM * There is insufficient memory available with the requested alignment. */ int posix_memalign(void **memptr, size_t alignment, size_t size) { void *retval; /* POSIX is silent on NULL == memptr !?! */ retval = malloc_zone_memalign(inline_malloc_default_zone(), alignment, size); if (retval == NULL) { // To avoid testing the alignment constraints redundantly, we'll rely on the // test made in malloc_zone_memalign to vet each request. Only if that test fails // and returns NULL, do we arrive here to detect the bogus alignment and give the // required EINVAL return. if (alignment < sizeof( void *) || // excludes 0 == alignment 0 != (alignment & (alignment - 1))) { // relies on sizeof(void *) being a power of two. return EINVAL; } return ENOMEM; } else { *memptr = retval; // Set iff allocation succeeded return 0; } } static malloc_zone_t * find_registered_purgeable_zone(void *ptr) { if (!ptr) return NULL; /* * Look for a zone which contains ptr. If that zone does not have the purgeable malloc flag * set, or the allocation is too small, do nothing. Otherwise, set the allocation volatile. * FIXME: for performance reasons, we should probably keep a separate list of purgeable zones * and only search those. */ size_t size = 0; malloc_zone_t *zone = find_registered_zone(ptr, &size); /* FIXME: would really like a zone->introspect->flags->purgeable check, but haven't determined * binary compatibility impact of changing the introspect struct yet. */ if (!zone) return NULL; /* Check to make sure pointer is page aligned and size is multiple of page size */ if ((size < vm_page_size) || ((size % vm_page_size) != 0)) return NULL; return zone; } void malloc_make_purgeable(void *ptr) { malloc_zone_t *zone = find_registered_purgeable_zone(ptr); if (!zone) return; int state = VM_PURGABLE_VOLATILE; vm_purgable_control(mach_task_self(), (vm_address_t)ptr, VM_PURGABLE_SET_STATE, &state); return; } /* Returns true if ptr is valid. Ignore the return value from vm_purgeable_control and only report * state. */ int malloc_make_nonpurgeable(void *ptr) { malloc_zone_t *zone = find_registered_purgeable_zone(ptr); if (!zone) return 0; int state = VM_PURGABLE_NONVOLATILE; vm_purgable_control(mach_task_self(), (vm_address_t)ptr, VM_PURGABLE_SET_STATE, &state); if (state == VM_PURGABLE_EMPTY) return EFAULT; return 0; } /********* Batch methods ************/ unsigned malloc_zone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, unsigned num_requested) { unsigned (*batch_malloc)(malloc_zone_t *, size_t, void **, unsigned) = zone-> batch_malloc; if (! batch_malloc) return 0; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } unsigned batched = batch_malloc(zone, size, results, num_requested); if (malloc_logger) { unsigned index = 0; while (index < batched) { malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)results[index], 0); index++; } } return batched; } void malloc_zone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) { if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } if (malloc_logger) { unsigned index = 0; while (index < num) { malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)to_be_freed[index], 0, 0, 0); index++; } } void (*batch_free)(malloc_zone_t *, void **, unsigned) = zone-> batch_free; if (batch_free) { batch_free(zone, to_be_freed, num); } else { void (*free_fun)(malloc_zone_t *, void *) = zone->free; while (num--) { void *ptr = *to_be_freed++; free_fun(zone, ptr); } } } /********* Functions for performance tools ************/ static kern_return_t _malloc_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr) { *ptr = (void *)address; return 0; } kern_return_t malloc_get_all_zones(task_t task, memory_reader_t reader, vm_address_t **addresses, unsigned *count) { // Note that the 2 following addresses are not correct if the address of the target is different from your own. This notably occurs if the address of System.framework is slid (e.g. different than at B & I ) vm_address_t remote_malloc_zones = (vm_address_t)&malloc_zones; vm_address_t remote_malloc_num_zones = (vm_address_t)&malloc_num_zones; kern_return_t err; vm_address_t zones_address; vm_address_t *zones_address_ref; unsigned num_zones; unsigned *num_zones_ref; if (!reader) reader = _malloc_default_reader; // printf("Read malloc_zones at address %p should be %p\n", &malloc_zones, malloc_zones); err = reader(task, remote_malloc_zones, sizeof(void *), (void **)&zones_address_ref); // printf("Read malloc_zones[%p]=%p\n", remote_malloc_zones, *zones_address_ref); if (err) { malloc_printf("*** malloc_get_all_zones: error reading zones_address at %p\n", (unsigned)remote_malloc_zones); return err; } zones_address = *zones_address_ref; // printf("Reading num_zones at address %p\n", remote_malloc_num_zones); err = reader(task, remote_malloc_num_zones, sizeof(unsigned), (void **)&num_zones_ref); if (err) { malloc_printf("*** malloc_get_all_zones: error reading num_zones at %p\n", (unsigned)remote_malloc_num_zones); return err; } num_zones = *num_zones_ref; // printf("Read malloc_num_zones[%p]=%d\n", remote_malloc_num_zones, num_zones); *count = num_zones; // printf("malloc_get_all_zones succesfully found %d zones\n", num_zones); err = reader(task, zones_address, sizeof(malloc_zone_t *) * num_zones, (void **)addresses); if (err) { malloc_printf("*** malloc_get_all_zones: error reading zones at %p\n", &zones_address); return err; } // printf("malloc_get_all_zones succesfully read %d zones\n", num_zones); return err; } /********* Debug helpers ************/ void malloc_zone_print_ptr_info(void *ptr) { malloc_zone_t *zone; if (!ptr) return; zone = malloc_zone_from_ptr(ptr); if (zone) { printf("ptr %p in registered zone %p\n", ptr, zone); } else { printf("ptr %p not in heap\n", ptr); } } boolean_t malloc_zone_check(malloc_zone_t *zone) { boolean_t ok = 1; if (!zone) { unsigned index = 0; while (index < malloc_num_zones) { zone = malloc_zones[index++]; if (!zone->introspect->check(zone)) ok = 0; } } else { ok = zone->introspect->check(zone); } return ok; } void malloc_zone_print(malloc_zone_t *zone, boolean_t verbose) { if (!zone) { unsigned index = 0; while (index < malloc_num_zones) { zone = malloc_zones[index++]; zone->introspect->print(zone, verbose); } } else { zone->introspect->print(zone, verbose); } } void malloc_zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { if (!zone) { memset(stats, 0, sizeof(*stats)); unsigned index = 0; while (index < malloc_num_zones) { zone = malloc_zones[index++]; malloc_statistics_t this_stats; zone->introspect->statistics(zone, &this_stats); stats->blocks_in_use += this_stats.blocks_in_use; stats->size_in_use += this_stats.size_in_use; stats->max_size_in_use += this_stats.max_size_in_use; stats->size_allocated += this_stats.size_allocated; } } else { zone->introspect->statistics(zone, stats); } } void malloc_zone_log(malloc_zone_t *zone, void *address) { if (!zone) { unsigned index = 0; while (index < malloc_num_zones) { zone = malloc_zones[index++]; zone->introspect->log(zone, address); } } else { zone->introspect->log(zone, address); } } /********* Misc other entry points ************/ static void DefaultMallocError(int x) { #if USE_SLEEP_RATHER_THAN_ABORT malloc_printf("*** error %d\n", x); sleep(3600); #else _SIMPLE_STRING b = _simple_salloc(); if (b) { _simple_sprintf(b, "*** error %d", x); malloc_printf("%s\n", _simple_string(b)); __crashreporter_info__ = _simple_string(b); } else { _malloc_printf(MALLOC_PRINTF_NOLOG, "*** error %d", x); __crashreporter_info__ = "*** DefaultMallocError called"; } abort(); #endif } void (* malloc_error(void (*func)(int)))(int) { return DefaultMallocError; } /* Stack logging fork-handling prototypes */ extern void __stack_logging_fork_prepare(); extern void __stack_logging_fork_parent(); extern void __stack_logging_fork_child(); void _malloc_fork_prepare() { /* Prepare the malloc module for a fork by insuring that no thread is in a malloc critical section */ unsigned index = 0; MALLOC_LOCK(); while (index < malloc_num_zones) { malloc_zone_t *zone = malloc_zones[index++]; zone->introspect->force_lock(zone); } __stack_logging_fork_prepare(); } void _malloc_fork_parent() { /* Called in the parent process after a fork() to resume normal operation. */ unsigned index = 0; __stack_logging_fork_parent(); MALLOC_UNLOCK(); while (index < malloc_num_zones) { malloc_zone_t *zone = malloc_zones[index++]; zone->introspect->force_unlock(zone); } } void _malloc_fork_child() { /* Called in the child process after a fork() to resume normal operation. In the MTASK case we also have to change memory inheritance so that the child does not share memory with the parent. */ unsigned index = 0; __stack_logging_fork_child(); MALLOC_UNLOCK(); while (index < malloc_num_zones) { malloc_zone_t *zone = malloc_zones[index++]; zone->introspect->force_unlock(zone); } } /* * A Glibc-like mstats() interface. * * Note that this interface really isn't very good, as it doesn't understand * that we may have multiple allocators running at once. We just massage * the result from malloc_zone_statistics in any case. */ struct mstats mstats(void) { malloc_statistics_t s; struct mstats m; malloc_zone_statistics(NULL, &s); m.bytes_total = s.size_allocated; m.chunks_used = s.blocks_in_use; m.bytes_used = s.size_in_use; m.chunks_free = 0; m.bytes_free = m.bytes_total - m.bytes_used; /* isn't this somewhat obvious? */ return(m); } /***************** OBSOLETE ENTRY POINTS ********************/ #if PHASE_OUT_OLD_MALLOC #error PHASE OUT THE FOLLOWING FUNCTIONS #else #warning PHASE OUT THE FOLLOWING FUNCTIONS #endif void set_malloc_singlethreaded(boolean_t single) { static boolean_t warned = 0; if (!warned) { #if PHASE_OUT_OLD_MALLOC malloc_printf("*** OBSOLETE: set_malloc_singlethreaded(%d)\n", single); #endif warned = 1; } } void malloc_singlethreaded() { static boolean_t warned = 0; if (!warned) { malloc_printf("*** OBSOLETE: malloc_singlethreaded()\n"); warned = 1; } } int malloc_debug(int level) { malloc_printf("*** OBSOLETE: malloc_debug()\n"); return 0; }