#include "config.h"
#if HAVE(STDINT_H)
#include <stdint.h>
#elif HAVE(INTTYPES_H)
#include <inttypes.h>
#else
#include <sys/types.h>
#endif
#if PLATFORM(WIN_OS)
#include "windows.h"
#else
#include <unistd.h>
#include <sys/mman.h>
#endif
#include <fcntl.h>
#include "TCSystemAlloc.h"
#include "TCSpinLock.h"
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
union MemoryAligner {
void* p;
double d;
size_t s;
};
static SpinLock spinlock = SPINLOCK_INITIALIZER;
static size_t pagesize = 0;
#ifndef WTF_CHANGES
static bool use_devmem = false;
#endif
static bool use_sbrk = false;
static bool use_mmap = true;
#if HAVE(VIRTUALALLOC)
static bool use_VirtualAlloc = true;
#endif
static bool devmem_failure = false;
static bool sbrk_failure = false;
static bool mmap_failure = false;
static bool VirtualAlloc_failure = false;
#ifndef WTF_CHANGES
DEFINE_int32(malloc_devmem_start, 0,
"Physical memory starting location in MB for /dev/mem allocation."
" Setting this to 0 disables /dev/mem allocation");
DEFINE_int32(malloc_devmem_limit, 0,
"Physical memory limit location in MB for /dev/mem allocation."
" Setting this to 0 means no limit.");
#else
static const int32_t FLAGS_malloc_devmem_start = 0;
static const int32_t FLAGS_malloc_devmem_limit = 0;
#endif
#if HAVE(SBRK)
static void* TrySbrk(size_t size, size_t alignment) {
size = ((size + alignment - 1) / alignment) * alignment;
void* result = sbrk(size);
if (result == reinterpret_cast<void*>(-1)) {
sbrk_failure = true;
return NULL;
}
uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
if ((ptr & (alignment-1)) == 0) return result;
size_t extra = alignment - (ptr & (alignment-1));
void* r2 = sbrk(extra);
if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
return reinterpret_cast<void*>(ptr + extra);
}
result = sbrk(size + alignment - 1);
if (result == reinterpret_cast<void*>(-1)) {
sbrk_failure = true;
return NULL;
}
ptr = reinterpret_cast<uintptr_t>(result);
if ((ptr & (alignment-1)) != 0) {
ptr += alignment - (ptr & (alignment-1));
}
return reinterpret_cast<void*>(ptr);
}
#endif
#if HAVE(MMAP)
static void* TryMmap(size_t size, size_t alignment) {
if (pagesize == 0) pagesize = getpagesize();
if (alignment < pagesize) alignment = pagesize;
size = ((size + alignment - 1) / alignment) * alignment;
size_t extra = 0;
if (alignment > pagesize) {
extra = alignment - pagesize;
}
void* result = mmap(NULL, size + extra,
PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0);
if (result == reinterpret_cast<void*>(MAP_FAILED)) {
mmap_failure = true;
return NULL;
}
uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
size_t adjust = 0;
if ((ptr & (alignment - 1)) != 0) {
adjust = alignment - (ptr & (alignment - 1));
}
if (adjust > 0) {
munmap(reinterpret_cast<void*>(ptr), adjust);
}
if (adjust < extra) {
munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
}
ptr += adjust;
return reinterpret_cast<void*>(ptr);
}
#endif
#if HAVE(VIRTUALALLOC)
static void* TryVirtualAlloc(size_t size, size_t alignment) {
if (pagesize == 0) {
SYSTEM_INFO system_info;
GetSystemInfo(&system_info);
pagesize = system_info.dwPageSize;
}
if (alignment < pagesize) alignment = pagesize;
size = ((size + alignment - 1) / alignment) * alignment;
size_t extra = 0;
if (alignment > pagesize) {
extra = alignment - pagesize;
}
void* result = VirtualAlloc(NULL, size + extra,
MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN,
PAGE_READWRITE);
if (result == NULL) {
VirtualAlloc_failure = true;
return NULL;
}
uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
size_t adjust = 0;
if ((ptr & (alignment - 1)) != 0) {
adjust = alignment - (ptr & (alignment - 1));
}
if (adjust > 0) {
VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT);
}
if (adjust < extra) {
VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT);
}
ptr += adjust;
return reinterpret_cast<void*>(ptr);
}
#endif
#ifndef WTF_CHANGES
static void* TryDevMem(size_t size, size_t alignment) {
static bool initialized = false;
static off_t physmem_base; static off_t physmem_limit; static int physmem_fd;
if (FLAGS_malloc_devmem_start == 0) {
return NULL;
}
if (!initialized) {
physmem_fd = open("/dev/mem", O_RDWR);
if (physmem_fd < 0) {
devmem_failure = true;
return NULL;
}
physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
initialized = true;
}
if (pagesize == 0) pagesize = getpagesize();
if (alignment < pagesize) alignment = pagesize;
size = ((size + alignment - 1) / alignment) * alignment;
size_t extra = 0;
if (alignment > pagesize) {
extra = alignment - pagesize;
}
if (physmem_limit != 0 && physmem_base + size + extra > physmem_limit) {
devmem_failure = true;
return NULL;
}
void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
MAP_SHARED, physmem_fd, physmem_base);
if (result == reinterpret_cast<void*>(MAP_FAILED)) {
devmem_failure = true;
return NULL;
}
uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
size_t adjust = 0;
if ((ptr & (alignment - 1)) != 0) {
adjust = alignment - (ptr & (alignment - 1));
}
if (adjust > 0) {
munmap(reinterpret_cast<void*>(ptr), adjust);
}
if (adjust < extra) {
munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
}
ptr += adjust;
physmem_base += adjust + size;
return reinterpret_cast<void*>(ptr);
}
#endif
void* TCMalloc_SystemAlloc(size_t size, size_t alignment) {
#ifndef WTF_CHANGES
if (TCMallocDebug::level >= TCMallocDebug::kVerbose) {
MESSAGE("TCMalloc_SystemAlloc(%" PRIuS ", %" PRIuS")\n",
size, alignment);
}
#endif
SpinLockHolder lock_holder(&spinlock);
if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
for (int i = 0; i < 2; i++) {
#ifndef WTF_CHANGES
if (use_devmem && !devmem_failure) {
void* result = TryDevMem(size, alignment);
if (result != NULL) return result;
}
#endif
#if HAVE(SBRK)
if (use_sbrk && !sbrk_failure) {
void* result = TrySbrk(size, alignment);
if (result != NULL) return result;
}
#endif
#if HAVE(MMAP)
if (use_mmap && !mmap_failure) {
void* result = TryMmap(size, alignment);
if (result != NULL) return result;
}
#endif
#if HAVE(VIRTUALALLOC)
if (use_VirtualAlloc && !VirtualAlloc_failure) {
void* result = TryVirtualAlloc(size, alignment);
if (result != NULL) return result;
}
#endif
devmem_failure = false;
sbrk_failure = false;
mmap_failure = false;
VirtualAlloc_failure = false;
}
return NULL;
}