#include <mach/mach_types.h>
#include <mach/machine.h>
#include <ppc/exception.h>
#include <ppc/machine_routines.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
#include <machine/pmap.h>
#include <vm/vm_kern.h>
#include <mach/vm_map.h>
static char *next = NULL; static int cur_routine = 0; static int matched;
int _cpu_capabilities = 0;
char *commPagePtr = NULL;
static void*
commpage_allocate( void )
{
extern vm_map_t com_region_map; vm_offset_t kernel_addr; vm_offset_t zero = 0;
vm_size_t size = _COMM_PAGE_AREA_USED; ipc_port_t handle;
if (com_region_map == NULL)
panic("commpage map is null");
if (vm_allocate(kernel_map,&kernel_addr,_COMM_PAGE_AREA_USED,VM_FLAGS_ANYWHERE))
panic("cannot allocate commpage");
if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+_COMM_PAGE_AREA_USED,VM_PROT_DEFAULT,FALSE))
panic("cannot wire commpage");
if (mach_make_memory_entry( kernel_map, &size, kernel_addr, VM_PROT_DEFAULT, &handle, NULL )) panic("cannot make entry for commpage");
if (vm_map_64( com_region_map, &zero, _COMM_PAGE_AREA_USED, 0, VM_FLAGS_FIXED, handle, 0, FALSE, VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE )) panic("cannot map commpage");
ipc_port_release(handle);
return (void*) kernel_addr; }
static void*
commpage_addr_of(
int addr_at_runtime )
{
return (void*) (commPagePtr + addr_at_runtime - _COMM_PAGE_BASE_ADDRESS);
}
static int
commpage_cpus( void )
{
int cpus;
cpus = ml_get_max_cpus();
if (cpus == 0)
panic("commpage cpus==0");
if (cpus > 0xFF)
cpus = 0xFF;
return cpus;
}
static void
commpage_init_cpu_capabilities( void )
{
struct per_proc_info *pp;
procFeatures *pfp;
int cpus;
int available;
pp = per_proc_info; pfp = &pp->pf; available = pfp->Available;
if ((available & pfAltivec) == 0) {
_cpu_capabilities &= ~kHasAltivec;
}
if (_cpu_capabilities & kDcbaAvailable) { _cpu_capabilities |= commpage_time_dcba(); }
cpus = commpage_cpus(); if (cpus == 1) _cpu_capabilities |= kUP;
_cpu_capabilities |= (cpus << kNumCPUsShift);
}
void
commpage_stuff(
int address,
void *source,
int length )
{
char *dest = commpage_addr_of(address);
if (dest < next)
panic("commpage overlap: %08 - %08X", dest, next);
bcopy((char*)source,dest,length);
next = (dest + length);
}
static void
commpage_change(
uint32_t *ptr,
int bytes,
uint32_t search_mask,
uint32_t search_pattern,
uint32_t new_mask,
uint32_t new_pattern,
int (*check)(uint32_t instruction) )
{
int words = bytes >> 2;
uint32_t word;
int found_one = 0;
while( (--words) >= 0 ) {
word = *ptr;
if ((word & search_mask)==search_pattern) {
if ((check==NULL) || (check(word))) { found_one = 1;
word &= ~new_mask;
word |= new_pattern;
*ptr = word;
}
}
ptr++;
}
if (!found_one)
panic("commpage opcode not found");
}
static int
commpage_onebit(
uint32_t mtcrf )
{
int x = (mtcrf >> 12) & 0xFF;
if (x==0)
panic("commpage bad mtcrf");
return (x & (x-1))==0 ? 1 : 0; }
static void
commpage_handle_dcbas(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if ((_cpu_capabilities & kDcbaAvailable) == 0) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0007FE; search = 0x7C0005EC; replace_mask = 0xFFFFFFFF; replace = 0x60000000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
}
}
static void
commpage_handle_syncs(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if (_NumCPUs() == 1) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0007FE; search = 0x7C0004AC; replace_mask = 0xFFFFFFFF; replace = 0x60000000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
}
}
static void
commpage_handle_mtcrfs(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if (_cpu_capabilities & k64Bit) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0007FE; search = 0x7C000120; replace_mask = 0x00100000; replace = 0x00100000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,commpage_onebit);
}
}
static void
commpage_stuff_routine(
commpage_descriptor *rd )
{
char *routine_code;
int must,cant;
if (rd->commpage_address != cur_routine) {
if ((cur_routine!=0) && (matched==0))
panic("commpage no match");
cur_routine = rd->commpage_address;
matched = 0;
}
must = _cpu_capabilities & rd->musthave;
cant = _cpu_capabilities & rd->canthave;
if ((must == rd->musthave) && (cant == 0)) {
if (matched)
panic("commpage duplicate matches");
matched = 1;
routine_code = ((char*)rd) + rd->code_offset;
commpage_stuff(rd->commpage_address,routine_code,rd->code_length);
if (rd->special & kCommPageDCBA)
commpage_handle_dcbas(rd->commpage_address,rd->code_length);
if (rd->special & kCommPageSYNC)
commpage_handle_syncs(rd->commpage_address,rd->code_length);
if (rd->special & kCommPageMTCRF)
commpage_handle_mtcrfs(rd->commpage_address,rd->code_length);
}
}
void
commpage_populate( void )
{
char c1;
short c2;
addr64_t c8;
static double two52 = 1048576.0 * 1048576.0 * 4096.0; static double ten6 = 1000000.0; commpage_descriptor **rd;
short version = _COMM_PAGE_THIS_VERSION;
commPagePtr = (char*) commpage_allocate();
commpage_init_cpu_capabilities();
commpage_stuff(_COMM_PAGE_VERSION,&version,2);
commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int));
c1 = (_cpu_capabilities & kHasAltivec) ? -1 : 0;
commpage_stuff(_COMM_PAGE_ALTIVEC,&c1,1);
c1 = (_cpu_capabilities & k64Bit) ? -1 : 0;
commpage_stuff(_COMM_PAGE_64_BIT,&c1,1);
if (_cpu_capabilities & kCache32)
c2 = 32;
else if (_cpu_capabilities & kCache64)
c2 = 64;
else if (_cpu_capabilities & kCache128)
c2 = 128;
commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
commpage_stuff(_COMM_PAGE_2_TO_52,&two52,8);
commpage_stuff(_COMM_PAGE_10_TO_6,&ten6,8);
c8 = 0; commpage_stuff(_COMM_PAGE_TIMEBASE,&c8,8);
commpage_stuff(_COMM_PAGE_TIMESTAMP,&c8,8);
commpage_stuff(_COMM_PAGE_SEC_PER_TICK,&c8,8);
extern commpage_descriptor mach_absolute_time_32;
extern commpage_descriptor mach_absolute_time_64;
extern commpage_descriptor spinlock_32_try_mp;
extern commpage_descriptor spinlock_32_try_up;
extern commpage_descriptor spinlock_64_try_mp;
extern commpage_descriptor spinlock_64_try_up;
extern commpage_descriptor spinlock_32_lock_mp;
extern commpage_descriptor spinlock_32_lock_up;
extern commpage_descriptor spinlock_64_lock_mp;
extern commpage_descriptor spinlock_64_lock_up;
extern commpage_descriptor spinlock_32_unlock_mp;
extern commpage_descriptor spinlock_32_unlock_up;
extern commpage_descriptor spinlock_64_unlock_mp;
extern commpage_descriptor spinlock_64_unlock_up;
extern commpage_descriptor pthread_getspecific_sprg3;
extern commpage_descriptor pthread_getspecific_uftrap;
extern commpage_descriptor gettimeofday_32;
extern commpage_descriptor gettimeofday_64;
extern commpage_descriptor commpage_flush_dcache;
extern commpage_descriptor commpage_flush_icache;
extern commpage_descriptor pthread_self_sprg3;
extern commpage_descriptor pthread_self_uftrap;
extern commpage_descriptor spinlock_relinquish;
extern commpage_descriptor bzero_32;
extern commpage_descriptor bzero_128;
extern commpage_descriptor bcopy_g3;
extern commpage_descriptor bcopy_g4;
extern commpage_descriptor bcopy_970;
extern commpage_descriptor bcopy_64;
extern commpage_descriptor bigcopy_970;
static commpage_descriptor *routines[] = {
&mach_absolute_time_32,
&mach_absolute_time_64,
&spinlock_32_try_mp,
&spinlock_32_try_up,
&spinlock_64_try_mp,
&spinlock_64_try_up,
&spinlock_32_lock_mp,
&spinlock_32_lock_up,
&spinlock_64_lock_mp,
&spinlock_64_lock_up,
&spinlock_32_unlock_mp,
&spinlock_32_unlock_up,
&spinlock_64_unlock_mp,
&spinlock_64_unlock_up,
&pthread_getspecific_sprg3,
&pthread_getspecific_uftrap,
&gettimeofday_32,
&gettimeofday_64,
&commpage_flush_dcache,
&commpage_flush_icache,
&pthread_self_sprg3,
&pthread_self_uftrap,
&spinlock_relinquish,
&bzero_32,
&bzero_128,
&bcopy_g3,
&bcopy_g4,
&bcopy_970,
&bcopy_64,
&bigcopy_970,
NULL };
for( rd = routines; *rd != NULL ; rd++ )
commpage_stuff_routine(*rd);
if (!matched)
panic("commpage no match on last routine");
if (next > (commPagePtr + _COMM_PAGE_AREA_USED))
panic("commpage overflow");
sync_cache_virtual((vm_offset_t) commPagePtr,_COMM_PAGE_AREA_USED);
}