#if defined __linux__ && !defined _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <fficonfig.h>
#include <ffi.h>
#include <ffi_common.h>
#ifdef __NetBSD__
#include <sys/param.h>
#endif
#if __NetBSD_Version__ - 0 >= 799007200
#include <sys/mman.h>
#include <stddef.h>
#include <unistd.h>
static const size_t overhead =
(sizeof(max_align_t) > sizeof(void *) + sizeof(size_t)) ?
sizeof(max_align_t)
: sizeof(void *) + sizeof(size_t);
#define ADD_TO_POINTER(p, d) ((void *)((uintptr_t)(p) + (d)))
void *
ffi_closure_alloc (size_t size, void **code)
{
static size_t page_size;
size_t rounded_size;
void *codeseg, *dataseg;
int prot;
if (!code)
return NULL;
if (!page_size)
page_size = sysconf(_SC_PAGESIZE);
rounded_size = (size + overhead + page_size - 1) & ~(page_size - 1);
prot = PROT_READ | PROT_WRITE | PROT_MPROTECT(PROT_EXEC);
dataseg = mmap(NULL, rounded_size, prot, MAP_ANON | MAP_PRIVATE, -1, 0);
if (dataseg == MAP_FAILED)
return NULL;
codeseg = mremap(dataseg, rounded_size, NULL, rounded_size, MAP_REMAPDUP);
if (codeseg == MAP_FAILED) {
munmap(dataseg, rounded_size);
return NULL;
}
if (mprotect(codeseg, rounded_size, PROT_READ | PROT_EXEC) == -1) {
munmap(codeseg, rounded_size);
munmap(dataseg, rounded_size);
return NULL;
}
memcpy(dataseg, &rounded_size, sizeof(rounded_size));
memcpy(ADD_TO_POINTER(dataseg, sizeof(size_t)), &codeseg, sizeof(void *));
*code = ADD_TO_POINTER(codeseg, overhead);
return ADD_TO_POINTER(dataseg, overhead);
}
void
ffi_closure_free (void *ptr)
{
void *codeseg, *dataseg;
size_t rounded_size;
dataseg = ADD_TO_POINTER(ptr, -overhead);
memcpy(&rounded_size, dataseg, sizeof(rounded_size));
memcpy(&codeseg, ADD_TO_POINTER(dataseg, sizeof(size_t)), sizeof(void *));
munmap(dataseg, rounded_size);
munmap(codeseg, rounded_size);
}
#else
#if !FFI_MMAP_EXEC_WRIT && !FFI_EXEC_TRAMPOLINE_TABLE
# if __linux__ && !defined(__ANDROID__)
# define FFI_MMAP_EXEC_WRIT 1
# define HAVE_MNTENT 1
# endif
# if defined(X86_WIN32) || defined(X86_WIN64) || defined(__OS2__)
# define FFI_MMAP_EXEC_WRIT 1
# endif
#endif
#if FFI_MMAP_EXEC_WRIT && !defined FFI_MMAP_EXEC_SELINUX
# if defined(__linux__) && !defined(__ANDROID__)
# define FFI_MMAP_EXEC_SELINUX 1
# endif
#endif
#if FFI_CLOSURES
#if FFI_EXEC_TRAMPOLINE_TABLE
#ifdef __MACH__
#include <assert.h>
#include <dispatch/dispatch.h>
#include <dlfcn.h>
#include <mach/mach.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#if __has_feature(ptrauth_calls)
#include <ptrauth.h>
#define sign_ptr(p) ptrauth_sign_unauthenticated(p, ptrauth_key_function_pointer, 0)
#define auth_ptr(p) ptrauth_auth_data(p, ptrauth_key_function_pointer, 0)
#else
#define sign_ptr(p) p
#define auth_ptr(p) p
#endif
typedef struct ffi_trampoline_table ffi_trampoline_table;
typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry;
struct ffi_trampoline_table
{
vm_address_t config_page;
uint16_t free_count;
ffi_trampoline_table_entry *free_list;
ffi_trampoline_table_entry *free_list_pool;
ffi_trampoline_table *prev;
ffi_trampoline_table *next;
};
struct ffi_trampoline_table_entry
{
void *(*trampoline) (void);
ffi_trampoline_table_entry *next;
};
#define FFI_TRAMPOLINE_COUNT (PAGE_MAX_SIZE / FFI_TRAMPOLINE_SIZE)
#ifdef FFI_TRAMPOLINE_WHOLE_DYLIB
#define FFI_TRAMPOLINE_ALLOCATION_PAGE_COUNT 3
#define FFI_TRAMPOLINE_PAGE_SEGMENT_OFFSET PAGE_MAX_SIZE
#else
#define FFI_TRAMPOLINE_ALLOCATION_PAGE_COUNT 2
#define FFI_TRAMPOLINE_PAGE_SEGMENT_OFFSET 0
#endif
static pthread_mutex_t ffi_trampoline_lock = PTHREAD_MUTEX_INITIALIZER;
static ffi_trampoline_table *ffi_trampoline_tables = NULL;
static ffi_trampoline_table *
ffi_trampoline_table_alloc (void)
{
ffi_trampoline_table *table;
vm_address_t config_page;
vm_address_t trampoline_page;
vm_address_t trampoline_page_template;
vm_prot_t cur_prot;
vm_prot_t max_prot;
kern_return_t kt;
uint16_t i;
config_page = 0x0;
kt = vm_allocate (mach_task_self (), &config_page, FFI_TRAMPOLINE_ALLOCATION_PAGE_COUNT * PAGE_MAX_SIZE,
VM_FLAGS_ANYWHERE);
if (kt != KERN_SUCCESS)
return NULL;
static dispatch_once_t trampoline_template_init_once;
static void *ffi_closure_trampoline_table_page;
dispatch_once(&trampoline_template_init_once, ^{
void * const trampoline_handle = dlopen("/usr/lib/libffi-trampolines.dylib", RTLD_NOW | RTLD_LOCAL | RTLD_FIRST);
assert(trampoline_handle);
ffi_closure_trampoline_table_page = dlsym(trampoline_handle, "ffi_closure_trampoline_table_page");
assert(ffi_closure_trampoline_table_page);
});
trampoline_page_template = (uintptr_t) auth_ptr((void*)ffi_closure_trampoline_table_page);
#ifdef __arm__
trampoline_page_template &= ~1UL;
#endif
vm_address_t trampoline_segment_template = trampoline_page_template - FFI_TRAMPOLINE_PAGE_SEGMENT_OFFSET;
vm_size_t trampoline_segment_size = (FFI_TRAMPOLINE_ALLOCATION_PAGE_COUNT - 1) * PAGE_MAX_SIZE;
vm_address_t trampoline_segment = config_page + PAGE_MAX_SIZE;
kt = vm_remap (mach_task_self(), &trampoline_segment, trampoline_segment_size, 0x0,
VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, mach_task_self(), trampoline_segment_template,
FALSE, &cur_prot, &max_prot, VM_INHERIT_SHARE);
if (kt != KERN_SUCCESS || !(cur_prot & VM_PROT_EXECUTE))
{
vm_deallocate (mach_task_self (), config_page, FFI_TRAMPOLINE_ALLOCATION_PAGE_COUNT * PAGE_MAX_SIZE);
return NULL;
}
trampoline_page = trampoline_segment + FFI_TRAMPOLINE_PAGE_SEGMENT_OFFSET;
table = calloc (1, sizeof (ffi_trampoline_table));
table->free_count = FFI_TRAMPOLINE_COUNT;
table->config_page = config_page;
table->free_list_pool =
calloc (FFI_TRAMPOLINE_COUNT, sizeof (ffi_trampoline_table_entry));
for (i = 0; i < table->free_count; i++)
{
ffi_trampoline_table_entry *entry = &table->free_list_pool[i];
entry->trampoline =
(void *) (trampoline_page + (i * FFI_TRAMPOLINE_SIZE));
#if __has_feature(ptrauth_calls)
entry->trampoline = ptrauth_sign_unauthenticated(entry->trampoline, ptrauth_key_function_pointer, 0);
#endif
if (i < table->free_count - 1)
entry->next = &table->free_list_pool[i + 1];
}
table->free_list = table->free_list_pool;
return table;
}
static void
ffi_trampoline_table_free (ffi_trampoline_table *table)
{
if (table->prev != NULL)
table->prev->next = table->next;
if (table->next != NULL)
table->next->prev = table->prev;
vm_deallocate (mach_task_self (), table->config_page, PAGE_MAX_SIZE * 2);
free (table->free_list_pool);
free (table);
}
void *
ffi_closure_alloc (size_t size, void **code)
{
ffi_closure *closure = malloc (size);
if (closure == NULL)
return NULL;
pthread_mutex_lock (&ffi_trampoline_lock);
ffi_trampoline_table *table = ffi_trampoline_tables;
if (table == NULL || table->free_list == NULL)
{
table = ffi_trampoline_table_alloc ();
if (table == NULL)
{
pthread_mutex_unlock (&ffi_trampoline_lock);
free (closure);
return NULL;
}
table->next = ffi_trampoline_tables;
if (table->next != NULL)
table->next->prev = table;
ffi_trampoline_tables = table;
}
ffi_trampoline_table_entry *entry = ffi_trampoline_tables->free_list;
ffi_trampoline_tables->free_list = entry->next;
ffi_trampoline_tables->free_count--;
entry->next = NULL;
pthread_mutex_unlock (&ffi_trampoline_lock);
*code = entry->trampoline;
closure->trampoline_table = table;
closure->trampoline_table_entry = entry;
return closure;
}
void
ffi_closure_free (void *ptr)
{
ffi_closure *closure = ptr;
pthread_mutex_lock (&ffi_trampoline_lock);
ffi_trampoline_table *table = closure->trampoline_table;
ffi_trampoline_table_entry *entry = closure->trampoline_table_entry;
entry->next = table->free_list;
table->free_list = entry;
table->free_count++;
if (table->free_count == FFI_TRAMPOLINE_COUNT
&& ffi_trampoline_tables != table)
{
ffi_trampoline_table_free (table);
}
else if (ffi_trampoline_tables != table)
{
table->prev = NULL;
table->next = ffi_trampoline_tables;
if (ffi_trampoline_tables != NULL)
ffi_trampoline_tables->prev = table;
ffi_trampoline_tables = table;
}
pthread_mutex_unlock (&ffi_trampoline_lock);
free (closure);
}
#endif
#elif FFI_MMAP_EXEC_WRIT
#define USE_LOCKS 1
#define USE_DL_PREFIX 1
#ifdef __GNUC__
#ifndef USE_BUILTIN_FFS
#define USE_BUILTIN_FFS 1
#endif
#endif
#define HAVE_MORECORE 0
#define HAVE_MREMAP 0
#define NO_MALLINFO 1
#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
#define DEFAULT_GRANULARITY ((size_t)malloc_getpagesize)
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include <string.h>
#include <stdio.h>
#if !defined(X86_WIN32) && !defined(X86_WIN64)
#ifdef HAVE_MNTENT
#include <mntent.h>
#endif
#include <sys/param.h>
#include <pthread.h>
#include <sys/mman.h>
#define LACKS_SYS_MMAN_H 1
#if FFI_MMAP_EXEC_SELINUX
#include <sys/statfs.h>
#include <stdlib.h>
static int selinux_enabled = -1;
static int
selinux_enabled_check (void)
{
struct statfs sfs;
FILE *f;
char *buf = NULL;
size_t len = 0;
if (statfs ("/selinux", &sfs) >= 0
&& (unsigned int) sfs.f_type == 0xf97cff8cU)
return 1;
f = fopen ("/proc/mounts", "r");
if (f == NULL)
return 0;
while (getline (&buf, &len, f) >= 0)
{
char *p = strchr (buf, ' ');
if (p == NULL)
break;
p = strchr (p + 1, ' ');
if (p == NULL)
break;
if (strncmp (p + 1, "selinuxfs ", 10) == 0)
{
free (buf);
fclose (f);
return 1;
}
}
free (buf);
fclose (f);
return 0;
}
#define is_selinux_enabled() (selinux_enabled >= 0 ? selinux_enabled \
: (selinux_enabled = selinux_enabled_check ()))
#else
#define is_selinux_enabled() 0
#endif
#ifdef FFI_MMAP_EXEC_EMUTRAMP_PAX
#include <stdlib.h>
static int emutramp_enabled = -1;
static int
emutramp_enabled_check (void)
{
char *buf = NULL;
size_t len = 0;
FILE *f;
int ret;
f = fopen ("/proc/self/status", "r");
if (f == NULL)
return 0;
ret = 0;
while (getline (&buf, &len, f) != -1)
if (!strncmp (buf, "PaX:", 4))
{
char emutramp;
if (sscanf (buf, "%*s %*c%c", &emutramp) == 1)
ret = (emutramp == 'E');
break;
}
free (buf);
fclose (f);
return ret;
}
#define is_emutramp_enabled() (emutramp_enabled >= 0 ? emutramp_enabled \
: (emutramp_enabled = emutramp_enabled_check ()))
#endif
#elif defined (__CYGWIN__) || defined(__INTERIX)
#include <sys/mman.h>
#define is_selinux_enabled() 0
#endif
#ifndef FFI_MMAP_EXEC_EMUTRAMP_PAX
#define is_emutramp_enabled() 0
#endif
static void *dlmalloc(size_t);
static void dlfree(void*);
static void *dlcalloc(size_t, size_t) MAYBE_UNUSED;
static void *dlrealloc(void *, size_t) MAYBE_UNUSED;
static void *dlmemalign(size_t, size_t) MAYBE_UNUSED;
static void *dlvalloc(size_t) MAYBE_UNUSED;
static int dlmallopt(int, int) MAYBE_UNUSED;
static size_t dlmalloc_footprint(void) MAYBE_UNUSED;
static size_t dlmalloc_max_footprint(void) MAYBE_UNUSED;
static void** dlindependent_calloc(size_t, size_t, void**) MAYBE_UNUSED;
static void** dlindependent_comalloc(size_t, size_t*, void**) MAYBE_UNUSED;
static void *dlpvalloc(size_t) MAYBE_UNUSED;
static int dlmalloc_trim(size_t) MAYBE_UNUSED;
static size_t dlmalloc_usable_size(void*) MAYBE_UNUSED;
static void dlmalloc_stats(void) MAYBE_UNUSED;
#if !(defined(X86_WIN32) || defined(X86_WIN64) || defined(__OS2__)) || defined (__CYGWIN__) || defined(__INTERIX)
static void *dlmmap(void *, size_t, int, int, int, off_t);
static int dlmunmap(void *, size_t);
#endif
#define mmap dlmmap
#define munmap dlmunmap
#include "dlmalloc.c"
#undef mmap
#undef munmap
#if !(defined(X86_WIN32) || defined(X86_WIN64) || defined(__OS2__)) || defined (__CYGWIN__) || defined(__INTERIX)
static pthread_mutex_t open_temp_exec_file_mutex = PTHREAD_MUTEX_INITIALIZER;
static int execfd = -1;
static size_t execsize = 0;
static int
open_temp_exec_file_name (char *name, int flags)
{
int fd;
#ifdef HAVE_MKOSTEMP
fd = mkostemp (name, flags);
#else
fd = mkstemp (name);
#endif
if (fd != -1)
unlink (name);
return fd;
}
static int
open_temp_exec_file_dir (const char *dir)
{
static const char suffix[] = "/ffiXXXXXX";
int lendir, flags;
char *tempname;
#ifdef O_TMPFILE
int fd;
#endif
#ifdef O_CLOEXEC
flags = O_CLOEXEC;
#else
flags = 0;
#endif
#ifdef O_TMPFILE
fd = open (dir, flags | O_RDWR | O_EXCL | O_TMPFILE, 0700);
if (fd != -1 || (errno != EINVAL && errno != EISDIR && errno != EOPNOTSUPP)) {
return fd;
} else {
errno = 0;
}
#endif
lendir = (int) strlen (dir);
tempname = __builtin_alloca (lendir + sizeof (suffix));
if (!tempname)
return -1;
memcpy (tempname, dir, lendir);
memcpy (tempname + lendir, suffix, sizeof (suffix));
return open_temp_exec_file_name (tempname, flags);
}
static int
open_temp_exec_file_env (const char *envvar)
{
const char *value = getenv (envvar);
if (!value)
return -1;
return open_temp_exec_file_dir (value);
}
#ifdef HAVE_MNTENT
static int
open_temp_exec_file_mnt (const char *mounts)
{
static const char *last_mounts;
static FILE *last_mntent;
if (mounts != last_mounts)
{
if (last_mntent)
endmntent (last_mntent);
last_mounts = mounts;
if (mounts)
last_mntent = setmntent (mounts, "r");
else
last_mntent = NULL;
}
if (!last_mntent)
return -1;
for (;;)
{
int fd;
struct mntent mnt;
char buf[MAXPATHLEN * 3];
if (getmntent_r (last_mntent, &mnt, buf, sizeof (buf)) == NULL)
return -1;
if (hasmntopt (&mnt, "ro")
|| hasmntopt (&mnt, "noexec")
|| access (mnt.mnt_dir, W_OK))
continue;
fd = open_temp_exec_file_dir (mnt.mnt_dir);
if (fd != -1)
return fd;
}
}
#endif
static struct
{
int (*func)(const char *);
const char *arg;
int repeat;
} open_temp_exec_file_opts[] = {
{ open_temp_exec_file_env, "TMPDIR", 0 },
{ open_temp_exec_file_dir, "/tmp", 0 },
{ open_temp_exec_file_dir, "/var/tmp", 0 },
{ open_temp_exec_file_dir, "/dev/shm", 0 },
{ open_temp_exec_file_env, "HOME", 0 },
#ifdef HAVE_MNTENT
{ open_temp_exec_file_mnt, "/etc/mtab", 1 },
{ open_temp_exec_file_mnt, "/proc/mounts", 1 },
#endif
};
static int open_temp_exec_file_opts_idx = 0;
static int
open_temp_exec_file_opts_next (void)
{
if (open_temp_exec_file_opts[open_temp_exec_file_opts_idx].repeat)
open_temp_exec_file_opts[open_temp_exec_file_opts_idx].func (NULL);
open_temp_exec_file_opts_idx++;
if (open_temp_exec_file_opts_idx
== (sizeof (open_temp_exec_file_opts)
/ sizeof (*open_temp_exec_file_opts)))
{
open_temp_exec_file_opts_idx = 0;
return 1;
}
return 0;
}
static int
open_temp_exec_file (void)
{
int fd;
do
{
fd = open_temp_exec_file_opts[open_temp_exec_file_opts_idx].func
(open_temp_exec_file_opts[open_temp_exec_file_opts_idx].arg);
if (!open_temp_exec_file_opts[open_temp_exec_file_opts_idx].repeat
|| fd == -1)
{
if (open_temp_exec_file_opts_next ())
break;
}
}
while (fd == -1);
return fd;
}
static int
allocate_space (int fd, off_t offset, off_t len)
{
static size_t page_size;
if (!page_size)
page_size = sysconf(_SC_PAGESIZE);
unsigned char buf[page_size];
memset (buf, 0, page_size);
while (len > 0)
{
off_t to_write = (len < page_size) ? len : page_size;
if (write (fd, buf, to_write) < to_write)
return -1;
len -= to_write;
}
return 0;
}
static void *
dlmmap_locked (void *start, size_t length, int prot, int flags, off_t offset)
{
void *ptr;
if (execfd == -1)
{
open_temp_exec_file_opts_idx = 0;
retry_open:
execfd = open_temp_exec_file ();
if (execfd == -1)
return MFAIL;
}
offset = execsize;
if (allocate_space (execfd, offset, length))
return MFAIL;
flags &= ~(MAP_PRIVATE | MAP_ANONYMOUS);
flags |= MAP_SHARED;
ptr = mmap (NULL, length, (prot & ~PROT_WRITE) | PROT_EXEC,
flags, execfd, offset);
if (ptr == MFAIL)
{
if (!offset)
{
close (execfd);
goto retry_open;
}
ftruncate (execfd, offset);
return MFAIL;
}
else if (!offset
&& open_temp_exec_file_opts[open_temp_exec_file_opts_idx].repeat)
open_temp_exec_file_opts_next ();
start = mmap (start, length, prot, flags, execfd, offset);
if (start == MFAIL)
{
munmap (ptr, length);
ftruncate (execfd, offset);
return start;
}
mmap_exec_offset ((char *)start, length) = (char*)ptr - (char*)start;
execsize += length;
return start;
}
static void *
dlmmap (void *start, size_t length, int prot,
int flags, int fd, off_t offset)
{
void *ptr;
assert (start == NULL && length % malloc_getpagesize == 0
&& prot == (PROT_READ | PROT_WRITE)
&& flags == (MAP_PRIVATE | MAP_ANONYMOUS)
&& fd == -1 && offset == 0);
if (execfd == -1 && is_emutramp_enabled ())
{
ptr = mmap (start, length, prot & ~PROT_EXEC, flags, fd, offset);
return ptr;
}
if (execfd == -1 && !is_selinux_enabled ())
{
ptr = mmap (start, length, prot | PROT_EXEC, flags, fd, offset);
if (ptr != MFAIL || (errno != EPERM && errno != EACCES))
return ptr;
}
if (execsize == 0 || execfd == -1)
{
pthread_mutex_lock (&open_temp_exec_file_mutex);
ptr = dlmmap_locked (start, length, prot, flags, offset);
pthread_mutex_unlock (&open_temp_exec_file_mutex);
return ptr;
}
return dlmmap_locked (start, length, prot, flags, offset);
}
static int
dlmunmap (void *start, size_t length)
{
msegmentptr seg = segment_holding (gm, start);
void *code;
if (seg && (code = add_segment_exec_offset (start, seg)) != start)
{
int ret = munmap (code, length);
if (ret)
return ret;
}
return munmap (start, length);
}
#if FFI_CLOSURE_FREE_CODE
static msegmentptr
segment_holding_code (mstate m, char* addr)
{
msegmentptr sp = &m->seg;
for (;;) {
if (addr >= add_segment_exec_offset (sp->base, sp)
&& addr < add_segment_exec_offset (sp->base, sp) + sp->size)
return sp;
if ((sp = sp->next) == 0)
return 0;
}
}
#endif
#endif
void *
ffi_closure_alloc (size_t size, void **code)
{
void *ptr;
if (!code)
return NULL;
ptr = dlmalloc (size);
if (ptr)
{
msegmentptr seg = segment_holding (gm, ptr);
*code = add_segment_exec_offset (ptr, seg);
}
return ptr;
}
void
ffi_closure_free (void *ptr)
{
#if FFI_CLOSURE_FREE_CODE
msegmentptr seg = segment_holding_code (gm, ptr);
if (seg)
ptr = sub_segment_exec_offset (ptr, seg);
#endif
dlfree (ptr);
}
# else
#include <stdlib.h>
void *
ffi_closure_alloc (size_t size, void **code)
{
if (!code)
return NULL;
return *code = malloc (size);
}
void
ffi_closure_free (void *ptr)
{
free (ptr);
}
# endif
#endif
#endif