#include <ffi.h>
#include <ffi_common.h>
#include <stdlib.h>
#include <stdio.h>
#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 1)
# define hidden __attribute__ ((visibility ("hidden")))
#else
# define hidden
#endif
extern void ffi_closure_SYSV(void);
extern void hidden ffi_closure_LINUX64(void);
enum {
FLAG_RETURNS_NOTHING = 1 << (31-30),
FLAG_RETURNS_FP = 1 << (31-29),
FLAG_RETURNS_64BITS = 1 << (31-28),
FLAG_ARG_NEEDS_COPY = 1 << (31- 7),
FLAG_FP_ARGUMENTS = 1 << (31- 6),
FLAG_4_GPR_ARGUMENTS = 1 << (31- 5),
FLAG_RETVAL_REFERENCE = 1 << (31- 4)
};
enum {
NUM_GPR_ARG_REGISTERS = 8,
NUM_FPR_ARG_REGISTERS = 8
};
enum { ASM_NEEDS_REGISTERS = 4 };
void ffi_prep_args_SYSV(extended_cif *ecif, unsigned *const stack)
{
const unsigned bytes = ecif->cif->bytes;
const unsigned flags = ecif->cif->flags;
unsigned *const stacktop = stack + (bytes / sizeof(unsigned));
unsigned *gpr_base = stacktop - ASM_NEEDS_REGISTERS - NUM_GPR_ARG_REGISTERS;
int intarg_count = 0;
double *fpr_base = (double *)gpr_base - NUM_FPR_ARG_REGISTERS;
int fparg_count = 0;
char *copy_space = ((flags & FLAG_FP_ARGUMENTS)
? (char *)fpr_base
: (char *)gpr_base);
unsigned *next_arg = stack + 2;
int i;
ffi_type **ptr;
double double_tmp;
void **p_argv;
size_t struct_copy_size;
unsigned gprvalue;
FFI_ASSERT(((unsigned)(char *)stack & 0xF) == 0);
FFI_ASSERT(((unsigned)(char *)copy_space & 0xF) == 0);
FFI_ASSERT(((unsigned)(char *)stacktop & 0xF) == 0);
FFI_ASSERT((bytes & 0xF) == 0);
FFI_ASSERT(copy_space >= (char *)next_arg);
if (flags & FLAG_RETVAL_REFERENCE)
{
*gpr_base++ = (unsigned long)(char *)ecif->rvalue;
intarg_count++;
}
p_argv = ecif->avalue;
for (ptr = ecif->cif->arg_types, i = ecif->cif->nargs;
i > 0;
i--, ptr++, p_argv++)
{
switch ((*ptr)->type)
{
case FFI_TYPE_FLOAT:
double_tmp = *(float *)*p_argv;
if (fparg_count >= NUM_FPR_ARG_REGISTERS)
{
*(float *)next_arg = (float)double_tmp;
next_arg += 1;
}
else
*fpr_base++ = double_tmp;
fparg_count++;
FFI_ASSERT(flags & FLAG_FP_ARGUMENTS);
break;
case FFI_TYPE_DOUBLE:
double_tmp = *(double *)*p_argv;
if (fparg_count >= NUM_FPR_ARG_REGISTERS)
{
if (intarg_count%2 != 0)
{
intarg_count++;
next_arg++;
}
*(double *)next_arg = double_tmp;
next_arg += 2;
}
else
*fpr_base++ = double_tmp;
fparg_count++;
FFI_ASSERT(flags & FLAG_FP_ARGUMENTS);
break;
case FFI_TYPE_UINT64:
case FFI_TYPE_SINT64:
if (intarg_count == NUM_GPR_ARG_REGISTERS-1)
intarg_count++;
if (intarg_count >= NUM_GPR_ARG_REGISTERS)
{
if (intarg_count%2 != 0)
{
intarg_count++;
next_arg++;
}
*(long long *)next_arg = *(long long *)*p_argv;
next_arg += 2;
}
else
{
if (intarg_count%2 != 0)
{
intarg_count ++;
gpr_base++;
}
*(long long *)gpr_base = *(long long *)*p_argv;
gpr_base += 2;
}
intarg_count += 2;
break;
case FFI_TYPE_STRUCT:
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
case FFI_TYPE_LONGDOUBLE:
#endif
struct_copy_size = ((*ptr)->size + 15) & ~0xF;
copy_space -= struct_copy_size;
memcpy(copy_space, (char *)*p_argv, (*ptr)->size);
gprvalue = (unsigned long)copy_space;
FFI_ASSERT(copy_space > (char *)next_arg);
FFI_ASSERT(flags & FLAG_ARG_NEEDS_COPY);
goto putgpr;
case FFI_TYPE_UINT8:
gprvalue = *(unsigned char *)*p_argv;
goto putgpr;
case FFI_TYPE_SINT8:
gprvalue = *(signed char *)*p_argv;
goto putgpr;
case FFI_TYPE_UINT16:
gprvalue = *(unsigned short *)*p_argv;
goto putgpr;
case FFI_TYPE_SINT16:
gprvalue = *(signed short *)*p_argv;
goto putgpr;
case FFI_TYPE_INT:
case FFI_TYPE_UINT32:
case FFI_TYPE_SINT32:
case FFI_TYPE_POINTER:
gprvalue = *(unsigned *)*p_argv;
putgpr:
if (intarg_count >= NUM_GPR_ARG_REGISTERS)
*next_arg++ = gprvalue;
else
*gpr_base++ = gprvalue;
intarg_count++;
break;
}
}
FFI_ASSERT(copy_space >= (char *)next_arg);
FFI_ASSERT(gpr_base <= stacktop - ASM_NEEDS_REGISTERS);
FFI_ASSERT((unsigned *)fpr_base
<= stacktop - ASM_NEEDS_REGISTERS - NUM_GPR_ARG_REGISTERS);
FFI_ASSERT(flags & FLAG_4_GPR_ARGUMENTS || intarg_count <= 4);
}
enum {
NUM_GPR_ARG_REGISTERS64 = 8,
NUM_FPR_ARG_REGISTERS64 = 13
};
enum { ASM_NEEDS_REGISTERS64 = 4 };
void hidden ffi_prep_args64(extended_cif *ecif, unsigned long *const stack)
{
const unsigned long bytes = ecif->cif->bytes;
const unsigned long flags = ecif->cif->flags;
unsigned long *const stacktop = stack + (bytes / sizeof(unsigned long));
unsigned long *const gpr_base = stacktop - ASM_NEEDS_REGISTERS64
- NUM_GPR_ARG_REGISTERS64;
unsigned long *const gpr_end = gpr_base + NUM_GPR_ARG_REGISTERS64;
unsigned long *const rest = stack + 6 + NUM_GPR_ARG_REGISTERS64;
unsigned long *next_arg = gpr_base;
double *fpr_base = (double *)gpr_base - NUM_FPR_ARG_REGISTERS64;
int fparg_count = 0;
int i, words;
ffi_type **ptr;
double double_tmp;
void **p_argv;
unsigned long gprvalue;
FFI_ASSERT(((unsigned long)(char *)stack & 0xF) == 0);
FFI_ASSERT(((unsigned long)(char *)stacktop & 0xF) == 0);
FFI_ASSERT((bytes & 0xF) == 0);
if (flags & FLAG_RETVAL_REFERENCE)
*next_arg++ = (unsigned long)(char *)ecif->rvalue;
p_argv = ecif->avalue;
for (ptr = ecif->cif->arg_types, i = ecif->cif->nargs;
i > 0;
i--, ptr++, p_argv++)
{
switch ((*ptr)->type)
{
case FFI_TYPE_FLOAT:
double_tmp = *(float *)*p_argv;
*(float *)next_arg = (float)double_tmp;
if (++next_arg == gpr_end)
next_arg = rest;
if (fparg_count < NUM_FPR_ARG_REGISTERS64)
*fpr_base++ = double_tmp;
fparg_count++;
FFI_ASSERT(flags & FLAG_FP_ARGUMENTS);
break;
case FFI_TYPE_DOUBLE:
double_tmp = *(double *)*p_argv;
*(double *)next_arg = double_tmp;
if (++next_arg == gpr_end)
next_arg = rest;
if (fparg_count < NUM_FPR_ARG_REGISTERS64)
*fpr_base++ = double_tmp;
fparg_count++;
FFI_ASSERT(flags & FLAG_FP_ARGUMENTS);
break;
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
case FFI_TYPE_LONGDOUBLE:
double_tmp = ((double *) *p_argv)[0];
*(double *) next_arg = double_tmp;
if (++next_arg == gpr_end)
next_arg = rest;
if (fparg_count < NUM_FPR_ARG_REGISTERS64)
*fpr_base++ = double_tmp;
fparg_count++;
double_tmp = ((double *) *p_argv)[1];
*(double *) next_arg = double_tmp;
if (++next_arg == gpr_end)
next_arg = rest;
if (fparg_count < NUM_FPR_ARG_REGISTERS64)
*fpr_base++ = double_tmp;
fparg_count++;
FFI_ASSERT(flags & FLAG_FP_ARGUMENTS);
break;
#endif
case FFI_TYPE_STRUCT:
words = ((*ptr)->size + 7) / 8;
if (next_arg >= gpr_base && next_arg + words > gpr_end)
{
size_t first = (char *) gpr_end - (char *) next_arg;
memcpy((char *) next_arg, (char *) *p_argv, first);
memcpy((char *) rest, (char *) *p_argv + first,
(*ptr)->size - first);
next_arg = (unsigned long *) ((char *) rest + words * 8 - first);
}
else
{
char *where = (char *) next_arg;
if ((*ptr)->size < 8)
where += 8 - (*ptr)->size;
memcpy (where, (char *) *p_argv, (*ptr)->size);
next_arg += words;
if (next_arg == gpr_end)
next_arg = rest;
}
break;
case FFI_TYPE_UINT8:
gprvalue = *(unsigned char *)*p_argv;
goto putgpr;
case FFI_TYPE_SINT8:
gprvalue = *(signed char *)*p_argv;
goto putgpr;
case FFI_TYPE_UINT16:
gprvalue = *(unsigned short *)*p_argv;
goto putgpr;
case FFI_TYPE_SINT16:
gprvalue = *(signed short *)*p_argv;
goto putgpr;
case FFI_TYPE_UINT32:
gprvalue = *(unsigned int *)*p_argv;
goto putgpr;
case FFI_TYPE_INT:
case FFI_TYPE_SINT32:
gprvalue = *(signed int *)*p_argv;
goto putgpr;
case FFI_TYPE_UINT64:
case FFI_TYPE_SINT64:
case FFI_TYPE_POINTER:
gprvalue = *(unsigned long *)*p_argv;
putgpr:
*next_arg++ = gprvalue;
if (next_arg == gpr_end)
next_arg = rest;
break;
}
}
FFI_ASSERT(flags & FLAG_4_GPR_ARGUMENTS
|| (next_arg >= gpr_base && next_arg <= gpr_base + 4));
}
ffi_status ffi_prep_cif_machdep(ffi_cif *cif)
{
int i;
ffi_type **ptr;
unsigned bytes;
int fparg_count = 0, intarg_count = 0;
unsigned flags = 0;
unsigned struct_copy_size = 0;
unsigned type = cif->rtype->type;
if (cif->abi != FFI_LINUX64)
{
bytes = (2 + ASM_NEEDS_REGISTERS) * sizeof(int);
bytes += NUM_GPR_ARG_REGISTERS * sizeof(int);
}
else
{
bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof(long);
bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof(long);
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
if (type == FFI_TYPE_LONGDOUBLE)
type = FFI_TYPE_DOUBLE;
#endif
}
switch (type)
{
case FFI_TYPE_DOUBLE:
flags |= FLAG_RETURNS_64BITS;
case FFI_TYPE_FLOAT:
flags |= FLAG_RETURNS_FP;
break;
case FFI_TYPE_UINT64:
case FFI_TYPE_SINT64:
flags |= FLAG_RETURNS_64BITS;
break;
case FFI_TYPE_STRUCT:
if (cif->abi != FFI_GCC_SYSV && cif->abi != FFI_LINUX64)
{
if (cif->rtype->size <= 4)
break;
else if (cif->rtype->size <= 8)
{
flags |= FLAG_RETURNS_64BITS;
break;
}
}
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
case FFI_TYPE_LONGDOUBLE:
#endif
intarg_count++;
flags |= FLAG_RETVAL_REFERENCE;
case FFI_TYPE_VOID:
flags |= FLAG_RETURNS_NOTHING;
break;
default:
break;
}
if (cif->abi != FFI_LINUX64)
for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
{
switch ((*ptr)->type)
{
case FFI_TYPE_FLOAT:
fparg_count++;
break;
case FFI_TYPE_DOUBLE:
fparg_count++;
if (fparg_count > NUM_FPR_ARG_REGISTERS
&& intarg_count%2 != 0)
intarg_count++;
break;
case FFI_TYPE_UINT64:
case FFI_TYPE_SINT64:
if (intarg_count == NUM_GPR_ARG_REGISTERS-1
|| intarg_count%2 != 0)
intarg_count++;
intarg_count += 2;
break;
case FFI_TYPE_STRUCT:
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
case FFI_TYPE_LONGDOUBLE:
#endif
struct_copy_size += ((*ptr)->size + 15) & ~0xF;
default:
intarg_count++;
break;
}
}
else
for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
{
switch ((*ptr)->type)
{
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
case FFI_TYPE_LONGDOUBLE:
fparg_count += 2;
intarg_count += 2;
break;
#endif
case FFI_TYPE_FLOAT:
case FFI_TYPE_DOUBLE:
fparg_count++;
intarg_count++;
break;
case FFI_TYPE_STRUCT:
intarg_count += ((*ptr)->size + 7) / 8;
break;
default:
intarg_count++;
break;
}
}
if (fparg_count != 0)
flags |= FLAG_FP_ARGUMENTS;
if (intarg_count > 4)
flags |= FLAG_4_GPR_ARGUMENTS;
if (struct_copy_size != 0)
flags |= FLAG_ARG_NEEDS_COPY;
if (cif->abi != FFI_LINUX64)
{
if (fparg_count != 0)
bytes += NUM_FPR_ARG_REGISTERS * sizeof(double);
if (intarg_count > NUM_GPR_ARG_REGISTERS)
bytes += (intarg_count - NUM_GPR_ARG_REGISTERS) * sizeof(int);
if (fparg_count > NUM_FPR_ARG_REGISTERS)
bytes += (fparg_count - NUM_FPR_ARG_REGISTERS) * sizeof(double);
}
else
{
if (fparg_count != 0)
bytes += NUM_FPR_ARG_REGISTERS64 * sizeof(double);
if (intarg_count > NUM_GPR_ARG_REGISTERS64)
bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof(long);
}
bytes = (bytes + 15) & ~0xF;
bytes += struct_copy_size;
cif->flags = flags;
cif->bytes = bytes;
return FFI_OK;
}
extern void ffi_call_SYSV( extended_cif *,
unsigned, unsigned,
unsigned *,
void (*fn)());
extern void hidden ffi_call_LINUX64( extended_cif *,
unsigned long, unsigned long,
unsigned long *,
void (*fn)());
void ffi_call( ffi_cif *cif,
void (*fn)(),
void *rvalue,
void **avalue)
{
extended_cif ecif;
ecif.cif = cif;
ecif.avalue = avalue;
if ((rvalue == NULL) &&
(cif->rtype->type == FFI_TYPE_STRUCT))
{
ecif.rvalue = alloca(cif->rtype->size);
}
else
ecif.rvalue = rvalue;
switch (cif->abi)
{
#ifndef POWERPC64
case FFI_SYSV:
case FFI_GCC_SYSV:
ffi_call_SYSV(&ecif, -cif->bytes,
cif->flags, ecif.rvalue, fn);
break;
#else
case FFI_LINUX64:
ffi_call_LINUX64(&ecif, -(long) cif->bytes,
cif->flags, ecif.rvalue, fn);
break;
#endif
default:
FFI_ASSERT(0);
break;
}
}
#ifndef POWERPC64
static void flush_icache(char *, int);
#define MIN_CACHE_LINE_SIZE 8
static void flush_icache(char * addr1, int size)
{
int i;
char * addr;
for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE) {
addr = addr1 + i;
__asm__ volatile ("icbi 0,%0;" "dcbf 0,%0;" : : "r"(addr) : "memory");
}
addr = addr1 + size - 1;
__asm__ volatile ("icbi 0,%0;" "dcbf 0,%0;" "sync;" "isync;" : : "r"(addr) : "memory");
}
#endif
ffi_status
ffi_prep_closure (ffi_closure* closure,
ffi_cif* cif,
void (*fun)(ffi_cif*, void*, void**, void*),
void *user_data)
{
#ifdef POWERPC64
void **tramp = (void **) &closure->tramp[0];
FFI_ASSERT (cif->abi == FFI_LINUX64);
memcpy (tramp, (char *) ffi_closure_LINUX64, 16);
tramp[2] = (void *) closure;
#else
unsigned int *tramp;
FFI_ASSERT (cif->abi == FFI_GCC_SYSV);
tramp = (unsigned int *) &closure->tramp[0];
tramp[0] = 0x7c0802a6;
tramp[1] = 0x4800000d;
tramp[4] = 0x7d6802a6;
tramp[5] = 0x7c0803a6;
tramp[6] = 0x800b0000;
tramp[7] = 0x816b0004;
tramp[8] = 0x7c0903a6;
tramp[9] = 0x4e800420;
*(void **) &tramp[2] = (void *)ffi_closure_SYSV;
*(void **) &tramp[3] = (void *)closure;
flush_icache(&closure->tramp[0],FFI_TRAMPOLINE_SIZE);
#endif
closure->cif = cif;
closure->fun = fun;
closure->user_data = user_data;
return FFI_OK;
}
typedef union
{
float f;
double d;
} ffi_dblfl;
int ffi_closure_helper_SYSV (ffi_closure*, void*, unsigned long*,
ffi_dblfl*, unsigned long*);
int
ffi_closure_helper_SYSV (ffi_closure* closure, void * rvalue,
unsigned long * pgr, ffi_dblfl * pfr,
unsigned long * pst)
{
void ** avalue;
ffi_type ** arg_types;
long i, avn;
long nf;
long ng;
ffi_cif * cif;
double temp;
cif = closure->cif;
avalue = alloca(cif->nargs * sizeof(void *));
nf = 0;
ng = 0;
if (cif->rtype->type == FFI_TYPE_STRUCT)
{
rvalue = (void *) *pgr;
ng++;
pgr++;
}
i = 0;
avn = cif->nargs;
arg_types = cif->arg_types;
while (i < avn)
{
switch (arg_types[i]->type)
{
case FFI_TYPE_SINT8:
case FFI_TYPE_UINT8:
if (ng < 8) {
avalue[i] = (((char *)pgr)+3);
ng++;
pgr++;
} else {
avalue[i] = (((char *)pst)+3);
pst++;
}
break;
case FFI_TYPE_SINT16:
case FFI_TYPE_UINT16:
if (ng < 8) {
avalue[i] = (((char *)pgr)+2);
ng++;
pgr++;
} else {
avalue[i] = (((char *)pst)+2);
pst++;
}
break;
case FFI_TYPE_SINT32:
case FFI_TYPE_UINT32:
case FFI_TYPE_POINTER:
if (ng < 8) {
avalue[i] = pgr;
ng++;
pgr++;
} else {
avalue[i] = pst;
pst++;
}
break;
case FFI_TYPE_STRUCT:
if (ng < 8) {
avalue[i] = (void *) *pgr;
ng++;
pgr++;
} else {
avalue[i] = (void *) *pst;
pst++;
}
break;
case FFI_TYPE_SINT64:
case FFI_TYPE_UINT64:
if (ng < 7) {
if (ng & 0x01) {
ng++;
pgr++;
}
avalue[i] = pgr;
ng+=2;
pgr+=2;
} else {
if (((long)pst) & 4) pst++;
avalue[i] = pst;
pst+=2;
}
break;
case FFI_TYPE_FLOAT:
if (nf < 8) {
temp = pfr->d;
pfr->f = (float)temp;
avalue[i] = pfr;
nf++;
pfr++;
} else {
avalue[i] = pst;
nf++;
pst+=1;
}
break;
case FFI_TYPE_DOUBLE:
if (nf < 8) {
avalue[i] = pfr;
nf++;
pfr++;
} else {
if (((long)pst) & 4) pst++;
avalue[i] = pst;
nf++;
pst+=2;
}
break;
default:
FFI_ASSERT(0);
}
i++;
}
(closure->fun) (cif, rvalue, avalue, closure->user_data);
return cif->rtype->type;
}
int hidden ffi_closure_helper_LINUX64 (ffi_closure*, void*, unsigned long*,
ffi_dblfl*);
int hidden
ffi_closure_helper_LINUX64 (ffi_closure *closure, void *rvalue,
unsigned long *pst, ffi_dblfl *pfr)
{
void **avalue;
ffi_type **arg_types;
long i, avn;
ffi_cif *cif;
ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
cif = closure->cif;
avalue = alloca (cif->nargs * sizeof (void *));
if (cif->rtype->type == FFI_TYPE_STRUCT)
{
rvalue = (void *) *pst;
pst++;
}
i = 0;
avn = cif->nargs;
arg_types = cif->arg_types;
while (i < avn)
{
switch (arg_types[i]->type)
{
case FFI_TYPE_SINT8:
case FFI_TYPE_UINT8:
avalue[i] = (char *) pst + 7;
pst++;
break;
case FFI_TYPE_SINT16:
case FFI_TYPE_UINT16:
avalue[i] = (char *) pst + 6;
pst++;
break;
case FFI_TYPE_SINT32:
case FFI_TYPE_UINT32:
avalue[i] = (char *) pst + 4;
pst++;
break;
case FFI_TYPE_SINT64:
case FFI_TYPE_UINT64:
case FFI_TYPE_POINTER:
avalue[i] = pst;
pst++;
break;
case FFI_TYPE_STRUCT:
if (arg_types[i]->size < 8)
avalue[i] = (char *) pst + 8 - arg_types[i]->size;
else
avalue[i] = pst;
pst += (arg_types[i]->size + 7) / 8;
break;
case FFI_TYPE_FLOAT:
if (pfr < end_pfr)
{
double temp = pfr->d;
pfr->f = (float) temp;
avalue[i] = pfr;
pfr++;
}
else
avalue[i] = pst;
pst++;
break;
case FFI_TYPE_DOUBLE:
if (pfr < end_pfr)
{
avalue[i] = pfr;
pfr++;
}
else
avalue[i] = pst;
pst++;
break;
#if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
case FFI_TYPE_LONGDOUBLE:
if (pfr + 1 < end_pfr)
{
avalue[i] = pfr;
pfr += 2;
}
else
{
if (pfr < end_pfr)
{
*pst = *(unsigned long *) pfr;
pfr++;
}
avalue[i] = pst;
}
pst += 2;
break;
#endif
default:
FFI_ASSERT(0);
}
i++;
}
(closure->fun) (cif, rvalue, avalue, closure->user_data);
return cif->rtype->type;
}