#include <mach/mach_types.h>
#include <mach/machine.h>
#include <mach/vm_map.h>
#include <i386/machine_routines.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
#include <machine/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <ipc/ipc_port.h>
extern vm_map_t com_region_map32;
static uintptr_t next = 0; static int cur_routine = 0; static int matched;
int _cpu_capabilities = 0;
char *commPagePtr = NULL;
static void*
commpage_allocate( void )
{
vm_offset_t kernel_addr; vm_offset_t zero = 0;
vm_size_t size = _COMM_PAGE_AREA_LENGTH;
vm_map_entry_t entry;
ipc_port_t handle;
if (com_region_map32 == NULL)
panic("commpage map is null");
if (vm_allocate(kernel_map,&kernel_addr,_COMM_PAGE_AREA_LENGTH,VM_FLAGS_ANYWHERE))
panic("cannot allocate commpage");
if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+_COMM_PAGE_AREA_LENGTH,VM_PROT_DEFAULT,FALSE))
panic("cannot wire commpage");
if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)
panic("cannot find commpage entry");
entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
if (mach_make_memory_entry( kernel_map, &size, kernel_addr, VM_PROT_DEFAULT, &handle, NULL )) panic("cannot make entry for commpage");
if (vm_map_64( com_region_map32, &zero, _COMM_PAGE_AREA_LENGTH, 0, VM_FLAGS_FIXED, handle, 0, FALSE, VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE )) panic("cannot map commpage");
ipc_port_release(handle);
return (void*) kernel_addr; }
static void*
commpage_addr_of(
int addr_at_runtime )
{
return (void*) ((uintptr_t)commPagePtr + addr_at_runtime - _COMM_PAGE_BASE_ADDRESS);
}
static int
commpage_cpus( void )
{
int cpus;
cpus = ml_get_max_cpus();
if (cpus == 0)
panic("commpage cpus==0");
if (cpus > 0xFF)
cpus = 0xFF;
return cpus;
}
static void
commpage_init_cpu_capabilities( void )
{
int bits;
int cpus;
ml_cpu_info_t cpu_info;
bits = 0;
ml_cpu_get_info(&cpu_info);
switch (cpu_info.vector_unit) {
case 5:
bits |= kHasSSE3;
case 4:
bits |= kHasSSE2;
case 3:
bits |= kHasSSE;
case 2:
bits |= kHasMMX;
default:
break;
}
switch (cpu_info.cache_line_size) {
case 128:
bits |= kCache128;
break;
case 64:
bits |= kCache64;
break;
case 32:
bits |= kCache32;
break;
default:
break;
}
cpus = commpage_cpus();
if (cpus == 1)
bits |= kUP;
bits |= (cpus << kNumCPUsShift);
bits |= kFastThreadLocalStorage;
_cpu_capabilities = bits; }
static void
commpage_stuff(
int address,
void *source,
int length )
{
void *dest = commpage_addr_of(address);
if ((uintptr_t)dest < next)
panic("commpage overlap at address 0x%x, 0x%x < 0x%x", address, dest, next);
bcopy(source,dest,length);
next = ((uintptr_t)dest + length);
}
static void
commpage_stuff2(
int address,
void *source,
int length )
{
commpage_stuff(address, source, length);
}
static void
commpage_stuff_routine(
commpage_descriptor *rd )
{
int must,cant;
if (rd->commpage_address != cur_routine) {
if ((cur_routine!=0) && (matched==0))
panic("commpage no match");
cur_routine = rd->commpage_address;
matched = 0;
}
must = _cpu_capabilities & rd->musthave;
cant = _cpu_capabilities & rd->canthave;
if ((must == rd->musthave) && (cant == 0)) {
if (matched)
panic("commpage duplicate matches");
matched = 1;
commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length);
}
}
#define COMMPAGE_DESC(name) commpage_ ## name
#define EXTERN_COMMPAGE_DESC(name) \
extern commpage_descriptor COMMPAGE_DESC(name)
EXTERN_COMMPAGE_DESC(compare_and_swap32_mp);
EXTERN_COMMPAGE_DESC(compare_and_swap32_up);
EXTERN_COMMPAGE_DESC(compare_and_swap64_mp);
EXTERN_COMMPAGE_DESC(compare_and_swap64_up);
EXTERN_COMMPAGE_DESC(atomic_add32_mp);
EXTERN_COMMPAGE_DESC(atomic_add32_up);
EXTERN_COMMPAGE_DESC(mach_absolute_time);
EXTERN_COMMPAGE_DESC(spin_lock_try_mp);
EXTERN_COMMPAGE_DESC(spin_lock_try_up);
EXTERN_COMMPAGE_DESC(spin_lock_mp);
EXTERN_COMMPAGE_DESC(spin_lock_up);
EXTERN_COMMPAGE_DESC(spin_unlock);
EXTERN_COMMPAGE_DESC(pthread_getspecific);
EXTERN_COMMPAGE_DESC(gettimeofday);
EXTERN_COMMPAGE_DESC(sys_flush_dcache);
EXTERN_COMMPAGE_DESC(sys_icache_invalidate);
EXTERN_COMMPAGE_DESC(pthread_self);
EXTERN_COMMPAGE_DESC(relinquish);
EXTERN_COMMPAGE_DESC(bit_test_and_set_mp);
EXTERN_COMMPAGE_DESC(bit_test_and_set_up);
EXTERN_COMMPAGE_DESC(bit_test_and_clear_mp);
EXTERN_COMMPAGE_DESC(bit_test_and_clear_up);
EXTERN_COMMPAGE_DESC(bzero_scalar);
EXTERN_COMMPAGE_DESC(bcopy_scalar);
EXTERN_COMMPAGE_DESC(nanotime);
static commpage_descriptor *routines[] = {
&COMMPAGE_DESC(compare_and_swap32_mp),
&COMMPAGE_DESC(compare_and_swap32_up),
&COMMPAGE_DESC(compare_and_swap64_mp),
&COMMPAGE_DESC(compare_and_swap64_up),
&COMMPAGE_DESC(atomic_add32_mp),
&COMMPAGE_DESC(atomic_add32_up),
&COMMPAGE_DESC(mach_absolute_time),
&COMMPAGE_DESC(spin_lock_try_mp),
&COMMPAGE_DESC(spin_lock_try_up),
&COMMPAGE_DESC(spin_lock_mp),
&COMMPAGE_DESC(spin_lock_up),
&COMMPAGE_DESC(spin_unlock),
&COMMPAGE_DESC(pthread_getspecific),
&COMMPAGE_DESC(gettimeofday),
&COMMPAGE_DESC(sys_flush_dcache),
&COMMPAGE_DESC(sys_icache_invalidate),
&COMMPAGE_DESC(pthread_self),
&COMMPAGE_DESC(relinquish),
&COMMPAGE_DESC(bit_test_and_set_mp),
&COMMPAGE_DESC(bit_test_and_set_up),
&COMMPAGE_DESC(bit_test_and_clear_mp),
&COMMPAGE_DESC(bit_test_and_clear_up),
&COMMPAGE_DESC(bzero_scalar),
&COMMPAGE_DESC(bcopy_scalar),
&COMMPAGE_DESC(nanotime),
NULL
};
void
commpage_populate( void )
{
short c2;
static double two52 = 1048576.0 * 1048576.0 * 4096.0; static double ten6 = 1000000.0; commpage_descriptor **rd;
short version = _COMM_PAGE_THIS_VERSION;
commPagePtr = (char *)commpage_allocate();
commpage_init_cpu_capabilities();
commpage_stuff2(_COMM_PAGE_VERSION,&version,sizeof(short));
commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,
sizeof(int));
if (_cpu_capabilities & kCache32)
c2 = 32;
else if (_cpu_capabilities & kCache64)
c2 = 64;
else if (_cpu_capabilities & kCache128)
c2 = 128;
commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
c2 = 32;
commpage_stuff2(_COMM_PAGE_2_TO_52,&two52,8);
commpage_stuff2(_COMM_PAGE_10_TO_6,&ten6,8);
for( rd = routines; *rd != NULL ; rd++ )
commpage_stuff_routine(*rd);
if (!matched)
panic("commpage no match on last routine");
if (next > (uintptr_t)_COMM_PAGE_END)
panic("commpage overflow: next = 0x%08x, commPagePtr = 0x%08x", next, (uintptr_t)commPagePtr);
pmap_commpage_init((vm_offset_t) commPagePtr, _COMM_PAGE_BASE_ADDRESS,
_COMM_PAGE_AREA_LENGTH/INTEL_PGBYTES);
}
#define NO_REORDERING asm volatile("" : : : "memory")
void
commpage_set_nanotime(commpage_nanotime_t *newp)
{
commpage_nanotime_t *cnp;
if (commPagePtr == NULL)
return;
cnp = (commpage_nanotime_t *)commpage_addr_of(_COMM_PAGE_NANOTIME_INFO);
cnp->nt_check_tsc = newp->nt_base_tsc; NO_REORDERING;
cnp->nt_shift = newp->nt_shift; NO_REORDERING;
cnp->nt_scale = newp->nt_scale; NO_REORDERING;
cnp->nt_base_ns = newp->nt_base_ns; NO_REORDERING;
cnp->nt_base_tsc = newp->nt_base_tsc;
}