#include <ffi.h>
#include <ffi_common.h>
#include <stdlib.h>
#include <stdint.h>
#if __riscv_float_abi_double
#define ABI_FLEN 64
#define ABI_FLOAT double
#elif __riscv_float_abi_single
#define ABI_FLEN 32
#define ABI_FLOAT float
#endif
#define NARGREG 8
#define STKALIGN 16
#define MAXCOPYARG (2 * sizeof(double))
typedef struct call_context
{
#if ABI_FLEN
ABI_FLOAT fa[8];
#endif
size_t a[8];
char frame[16];
} call_context;
typedef struct call_builder
{
call_context *aregs;
int used_integer;
int used_float;
size_t *used_stack;
} call_builder;
#if __SIZEOF_POINTER__ == 8
#define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT64)
#else
#define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT32)
#endif
#if ABI_FLEN
typedef struct {
char as_elements, type1, offset2, type2;
} float_struct_info;
#if ABI_FLEN >= 64
#define IS_FLOAT(type) ((type) >= FFI_TYPE_FLOAT && (type) <= FFI_TYPE_DOUBLE)
#else
#define IS_FLOAT(type) ((type) == FFI_TYPE_FLOAT)
#endif
static ffi_type **flatten_struct(ffi_type *in, ffi_type **out, ffi_type **out_end) {
int i;
if (out == out_end) return out;
if (in->type != FFI_TYPE_STRUCT) {
*(out++) = in;
} else {
for (i = 0; in->elements[i]; i++)
out = flatten_struct(in->elements[i], out, out_end);
}
return out;
}
static float_struct_info struct_passed_as_elements(call_builder *cb, ffi_type *top) {
float_struct_info ret = {0, 0, 0, 0};
ffi_type *fields[3];
int num_floats, num_ints;
int num_fields = flatten_struct(top, fields, fields + 3) - fields;
if (num_fields == 1) {
if (IS_FLOAT(fields[0]->type)) {
ret.as_elements = 1;
ret.type1 = fields[0]->type;
}
} else if (num_fields == 2) {
num_floats = IS_FLOAT(fields[0]->type) + IS_FLOAT(fields[1]->type);
num_ints = IS_INT(fields[0]->type) + IS_INT(fields[1]->type);
if (num_floats == 0 || num_floats + num_ints != 2)
return ret;
if (cb->used_float + num_floats > NARGREG || cb->used_integer + (2 - num_floats) > NARGREG)
return ret;
if (!IS_FLOAT(fields[0]->type) && !IS_FLOAT(fields[1]->type))
return ret;
ret.type1 = fields[0]->type;
ret.type2 = fields[1]->type;
ret.offset2 = FFI_ALIGN(fields[0]->size, fields[1]->alignment);
ret.as_elements = 1;
}
return ret;
}
#endif
static void marshal_atom(call_builder *cb, int type, void *data) {
size_t value = 0;
switch (type) {
case FFI_TYPE_UINT8: value = *(uint8_t *)data; break;
case FFI_TYPE_SINT8: value = *(int8_t *)data; break;
case FFI_TYPE_UINT16: value = *(uint16_t *)data; break;
case FFI_TYPE_SINT16: value = *(int16_t *)data; break;
case FFI_TYPE_UINT32: value = *(int32_t *)data; break;
case FFI_TYPE_SINT32: value = *(int32_t *)data; break;
#if __SIZEOF_POINTER__ == 8
case FFI_TYPE_UINT64: value = *(uint64_t *)data; break;
case FFI_TYPE_SINT64: value = *(int64_t *)data; break;
#endif
case FFI_TYPE_POINTER: value = *(size_t *)data; break;
#if ABI_FLEN >= 32
case FFI_TYPE_FLOAT:
asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(float *)data));
return;
#endif
#if ABI_FLEN >= 64
case FFI_TYPE_DOUBLE:
asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(double *)data));
return;
#endif
default: FFI_ASSERT(0); break;
}
if (cb->used_integer == NARGREG) {
*cb->used_stack++ = value;
} else {
cb->aregs->a[cb->used_integer++] = value;
}
}
static void unmarshal_atom(call_builder *cb, int type, void *data) {
size_t value;
switch (type) {
#if ABI_FLEN >= 32
case FFI_TYPE_FLOAT:
asm("" : "=f"(*(float *)data) : "0"(cb->aregs->fa[cb->used_float++]));
return;
#endif
#if ABI_FLEN >= 64
case FFI_TYPE_DOUBLE:
asm("" : "=f"(*(double *)data) : "0"(cb->aregs->fa[cb->used_float++]));
return;
#endif
}
if (cb->used_integer == NARGREG) {
value = *cb->used_stack++;
} else {
value = cb->aregs->a[cb->used_integer++];
}
switch (type) {
case FFI_TYPE_UINT8: *(uint8_t *)data = value; break;
case FFI_TYPE_SINT8: *(uint8_t *)data = value; break;
case FFI_TYPE_UINT16: *(uint16_t *)data = value; break;
case FFI_TYPE_SINT16: *(uint16_t *)data = value; break;
case FFI_TYPE_UINT32: *(uint32_t *)data = value; break;
case FFI_TYPE_SINT32: *(uint32_t *)data = value; break;
#if __SIZEOF_POINTER__ == 8
case FFI_TYPE_UINT64: *(uint64_t *)data = value; break;
case FFI_TYPE_SINT64: *(uint64_t *)data = value; break;
#endif
case FFI_TYPE_POINTER: *(size_t *)data = value; break;
default: FFI_ASSERT(0); break;
}
}
static void marshal(call_builder *cb, ffi_type *type, int var, void *data) {
size_t realign[2];
#if ABI_FLEN
if (!var && type->type == FFI_TYPE_STRUCT) {
float_struct_info fsi = struct_passed_as_elements(cb, type);
if (fsi.as_elements) {
marshal_atom(cb, fsi.type1, data);
if (fsi.offset2)
marshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
return;
}
}
if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
marshal_atom(cb, type->type, data);
return;
}
#endif
if (type->size > 2 * __SIZEOF_POINTER__) {
marshal_atom(cb, FFI_TYPE_POINTER, &data);
} else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
marshal_atom(cb, type->type, data);
} else {
if (type->alignment > __SIZEOF_POINTER__) {
if (var)
cb->used_integer = FFI_ALIGN(cb->used_integer, 2);
cb->used_stack = (size_t *)FFI_ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
}
memcpy(realign, data, type->size);
if (type->size > 0)
marshal_atom(cb, FFI_TYPE_POINTER, realign);
if (type->size > __SIZEOF_POINTER__)
marshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
}
}
static void *unmarshal(call_builder *cb, ffi_type *type, int var, void *data) {
size_t realign[2];
void *pointer;
#if ABI_FLEN
if (!var && type->type == FFI_TYPE_STRUCT) {
float_struct_info fsi = struct_passed_as_elements(cb, type);
if (fsi.as_elements) {
unmarshal_atom(cb, fsi.type1, data);
if (fsi.offset2)
unmarshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
return data;
}
}
if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
unmarshal_atom(cb, type->type, data);
return data;
}
#endif
if (type->size > 2 * __SIZEOF_POINTER__) {
unmarshal_atom(cb, FFI_TYPE_POINTER, (char*)&pointer);
return pointer;
} else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
unmarshal_atom(cb, type->type, data);
return data;
} else {
if (type->alignment > __SIZEOF_POINTER__) {
if (var)
cb->used_integer = FFI_ALIGN(cb->used_integer, 2);
cb->used_stack = (size_t *)FFI_ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
}
if (type->size > 0)
unmarshal_atom(cb, FFI_TYPE_POINTER, realign);
if (type->size > __SIZEOF_POINTER__)
unmarshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
memcpy(data, realign, type->size);
return data;
}
}
static int passed_by_ref(call_builder *cb, ffi_type *type, int var) {
#if ABI_FLEN
if (!var && type->type == FFI_TYPE_STRUCT) {
float_struct_info fsi = struct_passed_as_elements(cb, type);
if (fsi.as_elements) return 0;
}
#endif
return type->size > 2 * __SIZEOF_POINTER__;
}
ffi_status ffi_prep_cif_machdep(ffi_cif *cif) {
cif->riscv_nfixedargs = cif->nargs;
return FFI_OK;
}
ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs) {
cif->riscv_nfixedargs = nfixedargs;
return FFI_OK;
}
extern void ffi_call_asm (void *stack, struct call_context *regs,
void (*fn) (void), void *closure) FFI_HIDDEN;
static void
ffi_call_int (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue,
void *closure)
{
size_t arg_bytes = cif->nargs <= 3 ? 0 :
FFI_ALIGN(2 * sizeof(size_t) * (cif->nargs - 3), STKALIGN);
size_t rval_bytes = 0;
if (rvalue == NULL && cif->rtype->size > 2*__SIZEOF_POINTER__)
rval_bytes = FFI_ALIGN(cif->rtype->size, STKALIGN);
size_t alloc_size = arg_bytes + rval_bytes + sizeof(call_context);
size_t alloc_base;
if (_Alignof(max_align_t) >= STKALIGN) {
alloc_base = (size_t)alloca(alloc_size);
} else {
alloc_base = FFI_ALIGN(alloca(alloc_size + STKALIGN - 1), STKALIGN);
}
if (rval_bytes)
rvalue = (void*)(alloc_base + arg_bytes);
call_builder cb;
cb.used_float = cb.used_integer = 0;
cb.aregs = (call_context*)(alloc_base + arg_bytes + rval_bytes);
cb.used_stack = (void*)alloc_base;
int return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
if (return_by_ref)
marshal(&cb, &ffi_type_pointer, 0, &rvalue);
int i;
for (i = 0; i < cif->nargs; i++)
marshal(&cb, cif->arg_types[i], i >= cif->riscv_nfixedargs, avalue[i]);
ffi_call_asm ((void *) alloc_base, cb.aregs, fn, closure);
cb.used_float = cb.used_integer = 0;
if (!return_by_ref && rvalue)
unmarshal(&cb, cif->rtype, 0, rvalue);
}
void
ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
{
ffi_call_int(cif, fn, rvalue, avalue, NULL);
}
void
ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
void **avalue, void *closure)
{
ffi_call_int(cif, fn, rvalue, avalue, closure);
}
extern void ffi_closure_asm(void) FFI_HIDDEN;
ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, void *codeloc)
{
uint32_t *tramp = (uint32_t *) &closure->tramp[0];
uint64_t fn = (uint64_t) (uintptr_t) ffi_closure_asm;
if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
return FFI_BAD_ABI;
tramp[0] = 0x00000317;
#if __SIZEOF_POINTER__ == 8
tramp[1] = 0x01033383;
#else
tramp[1] = 0x01032383;
#endif
tramp[2] = 0x00038067;
tramp[3] = 0x00000013;
tramp[4] = fn;
tramp[5] = fn >> 32;
closure->cif = cif;
closure->fun = fun;
closure->user_data = user_data;
__builtin___clear_cache(codeloc, codeloc + FFI_TRAMPOLINE_SIZE);
return FFI_OK;
}
extern void ffi_go_closure_asm (void) FFI_HIDDEN;
ffi_status
ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
void (*fun) (ffi_cif *, void *, void **, void *))
{
if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
return FFI_BAD_ABI;
closure->tramp = (void *) ffi_go_closure_asm;
closure->cif = cif;
closure->fun = fun;
return FFI_OK;
}
void FFI_HIDDEN
ffi_closure_inner (ffi_cif *cif,
void (*fun) (ffi_cif *, void *, void **, void *),
void *user_data,
size_t *stack, call_context *aregs)
{
void **avalue = alloca(cif->nargs * sizeof(void*));
char *astorage = alloca(cif->nargs * MAXCOPYARG);
void *rvalue;
call_builder cb;
int return_by_ref;
int i;
cb.aregs = aregs;
cb.used_integer = cb.used_float = 0;
cb.used_stack = stack;
return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
if (return_by_ref)
unmarshal(&cb, &ffi_type_pointer, 0, &rvalue);
else
rvalue = alloca(cif->rtype->size);
for (i = 0; i < cif->nargs; i++)
avalue[i] = unmarshal(&cb, cif->arg_types[i],
i >= cif->riscv_nfixedargs, astorage + i*MAXCOPYARG);
fun (cif, rvalue, avalue, user_data);
if (!return_by_ref && cif->rtype->type != FFI_TYPE_VOID) {
cb.used_integer = cb.used_float = 0;
marshal(&cb, cif->rtype, 0, rvalue);
}
}