#include <mach/mach_types.h>
#include <mach/machine.h>
#include <mach/vm_map.h>
#include <ppc/exception.h>
#include <ppc/machine_routines.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>
#include <machine/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <ipc/ipc_port.h>
extern vm_map_t com_region_map32; extern vm_map_t com_region_map64;
char *commPagePtr32 = NULL; char *commPagePtr64 = NULL; int _cpu_capabilities = 0;
static char *next; static int cur_routine; static int matched; static char *commPagePtr;
extern commpage_descriptor compare_and_swap32_on32;
extern commpage_descriptor compare_and_swap32_on64;
extern commpage_descriptor compare_and_swap64;
extern commpage_descriptor atomic_enqueue32;
extern commpage_descriptor atomic_enqueue64;
extern commpage_descriptor atomic_dequeue32_on32;
extern commpage_descriptor atomic_dequeue32_on64;
extern commpage_descriptor atomic_dequeue64;
extern commpage_descriptor memory_barrier_up;
extern commpage_descriptor memory_barrier_mp32;
extern commpage_descriptor memory_barrier_mp64;
extern commpage_descriptor atomic_add32;
extern commpage_descriptor atomic_add64;
extern commpage_descriptor mach_absolute_time_32;
extern commpage_descriptor mach_absolute_time_64;
extern commpage_descriptor mach_absolute_time_lp64;
extern commpage_descriptor spinlock_32_try_mp;
extern commpage_descriptor spinlock_32_try_up;
extern commpage_descriptor spinlock_64_try_mp;
extern commpage_descriptor spinlock_64_try_up;
extern commpage_descriptor spinlock_32_lock_mp;
extern commpage_descriptor spinlock_32_lock_up;
extern commpage_descriptor spinlock_64_lock_mp;
extern commpage_descriptor spinlock_64_lock_up;
extern commpage_descriptor spinlock_32_unlock_mp;
extern commpage_descriptor spinlock_32_unlock_up;
extern commpage_descriptor spinlock_64_unlock_mp;
extern commpage_descriptor spinlock_64_unlock_up;
extern commpage_descriptor pthread_getspecific_sprg3_32;
extern commpage_descriptor pthread_getspecific_sprg3_64;
extern commpage_descriptor pthread_getspecific_uftrap;
extern commpage_descriptor gettimeofday_32;
extern commpage_descriptor gettimeofday_g5_32;
extern commpage_descriptor gettimeofday_g5_64;
extern commpage_descriptor commpage_flush_dcache;
extern commpage_descriptor commpage_flush_icache;
extern commpage_descriptor pthread_self_sprg3;
extern commpage_descriptor pthread_self_uftrap;
extern commpage_descriptor spinlock_relinquish;
extern commpage_descriptor bzero_32;
extern commpage_descriptor bzero_128;
extern commpage_descriptor bcopy_g3;
extern commpage_descriptor bcopy_g4;
extern commpage_descriptor bcopy_970;
extern commpage_descriptor bcopy_64;
extern commpage_descriptor compare_and_swap32_on32b;
extern commpage_descriptor compare_and_swap32_on64b;
extern commpage_descriptor compare_and_swap64b;
extern commpage_descriptor memset_64;
extern commpage_descriptor memset_g3;
extern commpage_descriptor memset_g4;
extern commpage_descriptor memset_g5;
extern commpage_descriptor bigcopy_970;
static commpage_descriptor *routines[] = {
&compare_and_swap32_on32,
&compare_and_swap32_on64,
&compare_and_swap64,
&atomic_enqueue32,
&atomic_enqueue64,
&atomic_dequeue32_on32,
&atomic_dequeue32_on64,
&atomic_dequeue64,
&memory_barrier_up,
&memory_barrier_mp32,
&memory_barrier_mp64,
&atomic_add32,
&atomic_add64,
&mach_absolute_time_32,
&mach_absolute_time_64,
&mach_absolute_time_lp64,
&spinlock_32_try_mp,
&spinlock_32_try_up,
&spinlock_64_try_mp,
&spinlock_64_try_up,
&spinlock_32_lock_mp,
&spinlock_32_lock_up,
&spinlock_64_lock_mp,
&spinlock_64_lock_up,
&spinlock_32_unlock_mp,
&spinlock_32_unlock_up,
&spinlock_64_unlock_mp,
&spinlock_64_unlock_up,
&pthread_getspecific_sprg3_32,
&pthread_getspecific_sprg3_64,
&pthread_getspecific_uftrap,
&gettimeofday_32,
&gettimeofday_g5_32,
&gettimeofday_g5_64,
&commpage_flush_dcache,
&commpage_flush_icache,
&pthread_self_sprg3,
&pthread_self_uftrap,
&spinlock_relinquish,
&bzero_32,
&bzero_128,
&bcopy_g3,
&bcopy_g4,
&bcopy_970,
&bcopy_64,
&compare_and_swap32_on32b,
&compare_and_swap32_on64b,
&compare_and_swap64b,
&memset_64,
&memset_g3,
&memset_g4,
&memset_g5,
&bigcopy_970,
NULL };
static void*
commpage_allocate(
vm_map_t submap ) {
vm_offset_t kernel_addr; vm_offset_t zero = 0;
vm_size_t size = _COMM_PAGE_AREA_USED; vm_map_entry_t entry;
ipc_port_t handle;
if (submap == NULL)
panic("commpage submap is null");
if (vm_allocate(kernel_map,&kernel_addr,_COMM_PAGE_AREA_USED,VM_FLAGS_ANYWHERE))
panic("cannot allocate commpage");
if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+_COMM_PAGE_AREA_USED,VM_PROT_DEFAULT,FALSE))
panic("cannot wire commpage");
if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)
panic("cannot find commpage entry");
entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
if (mach_make_memory_entry( kernel_map, &size, kernel_addr, VM_PROT_DEFAULT, &handle, NULL )) panic("cannot make entry for commpage");
if (vm_map_64( submap, &zero, _COMM_PAGE_AREA_USED, 0, VM_FLAGS_FIXED, handle, 0, FALSE, VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE )) panic("cannot map commpage");
ipc_port_release(handle);
return (void*) kernel_addr; }
static void*
commpage_addr_of(
int addr_at_runtime )
{
return (void*) (commPagePtr + addr_at_runtime - _COMM_PAGE_BASE_ADDRESS);
}
static int
commpage_cpus( void )
{
int cpus;
cpus = ml_get_max_cpus();
if (cpus == 0)
panic("commpage cpus==0");
if (cpus > 0xFF)
cpus = 0xFF;
return cpus;
}
static void
commpage_init_cpu_capabilities( void )
{
procFeatures *pfp;
int cpus;
int available;
pfp = &(PerProcTable[0].ppe_vaddr->pf); available = pfp->Available;
if ((available & pfAltivec) == 0) {
_cpu_capabilities &= ~kHasAltivec;
}
if (_cpu_capabilities & kDcbaAvailable) { _cpu_capabilities |= commpage_time_dcba(); }
cpus = commpage_cpus(); if (cpus == 1) _cpu_capabilities |= kUP;
_cpu_capabilities |= (cpus << kNumCPUsShift);
if (_cpu_capabilities & k64Bit) _cpu_capabilities |= kFastThreadLocalStorage;
}
static void
commpage_stuff(
int address,
const void *source,
int length )
{
char *dest = commpage_addr_of(address);
if (dest < next)
panic("commpage overlap: %08 - %08X", dest, next);
bcopy((const char*)source,dest,length);
next = (dest + length);
}
static void
commpage_change(
uint32_t *ptr,
int bytes,
uint32_t search_mask,
uint32_t search_pattern,
uint32_t new_mask,
uint32_t new_pattern,
int (*check)(uint32_t instruction) )
{
int words = bytes >> 2;
uint32_t word;
while( (--words) >= 0 ) {
word = *ptr;
if ((word & search_mask)==search_pattern) {
if ((check==NULL) || (check(word))) { word &= ~new_mask;
word |= new_pattern;
*ptr = word;
}
}
ptr++;
}
}
static int
commpage_onebit(
uint32_t mtcrf )
{
int x = (mtcrf >> 12) & 0xFF;
if (x==0)
panic("commpage bad mtcrf");
return (x & (x-1))==0 ? 1 : 0; }
static int
commpage_srwi(
uint32_t rlwinm )
{
int sh = (rlwinm >> 11) & 0x1F; int mb = (rlwinm >> 6 ) & 0x1F;
return (sh + mb) == 32; }
static void
commpage_handle_dcbas(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if ( (_cpu_capabilities & kDcbaRecommended) == 0 ) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0007FE; search = 0x7C0005EC; replace_mask = 0xFFFFFFFF; replace = 0x60000000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
}
}
static void
commpage_handle_syncs(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if (_NumCPUs() == 1) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0005FE; search = 0x7C0004AC; replace_mask = 0xFFFFFFFF; replace = 0x60000000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
}
}
static void
commpage_handle_isyncs(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if (_NumCPUs() == 1) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0007FE; search = 0x4C00012C; replace_mask = 0xFFFFFFFF; replace = 0x60000000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
}
}
static void
commpage_handle_mtcrfs(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
if (_cpu_capabilities & k64Bit) {
ptr = commpage_addr_of(address);
search_mask = 0xFC0007FE; search = 0x7C000120; replace_mask = 0x00100000; replace = 0x00100000;
commpage_change(ptr,length,search_mask,search,replace_mask,replace,commpage_onebit);
}
}
static void
commpage_port_32_to_64(
int address,
int length )
{
uint32_t *ptr, search_mask, search, replace_mask, replace;
ptr = commpage_addr_of(address);
search_mask = 0xFC2007FE; search = 0x7C000000; replace_mask = 0x00200000; replace = 0x00200000; commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
search_mask = 0xFC2007FE; search = 0x7C000040; replace_mask = 0x00200000; replace = 0x00200000; commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
search_mask = 0xFC200000; search = 0x28000000; replace_mask = 0x00200000; replace = 0x00200000; commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
search_mask = 0xFC200000; search = 0x2C000000; replace_mask = 0x00200000; replace = 0x00200000; commpage_change(ptr,length,search_mask,search,replace_mask,replace,NULL);
search_mask = 0xFC00003E; search = 0x5400003E; replace_mask = 0xFC00003E; replace = 0x78000002; commpage_change(ptr,length,search_mask,search,replace_mask,replace,commpage_srwi);
}
static void
commpage_stuff_routine(
commpage_descriptor *rd,
int mode ) {
char *routine_code;
int must,cant;
if ( (rd->special & mode) == 0 ) return;
if (rd->commpage_address != cur_routine) {
if ((cur_routine!=0) && (matched==0))
panic("commpage no match for last, next address %08x", rd->commpage_address);
cur_routine = rd->commpage_address;
matched = 0;
}
must = _cpu_capabilities & rd->musthave;
cant = _cpu_capabilities & rd->canthave;
if ((must == rd->musthave) && (cant == 0)) {
if (matched)
panic("commpage multiple matches for address %08x", rd->commpage_address);
matched = 1;
routine_code = ((char*)rd) + rd->code_offset;
commpage_stuff(rd->commpage_address,routine_code,rd->code_length);
if (rd->special & kCommPageDCBA)
commpage_handle_dcbas(rd->commpage_address,rd->code_length);
if (rd->special & kCommPageSYNC)
commpage_handle_syncs(rd->commpage_address,rd->code_length);
if (rd->special & kCommPageISYNC)
commpage_handle_isyncs(rd->commpage_address,rd->code_length);
if (rd->special & kCommPageMTCRF)
commpage_handle_mtcrfs(rd->commpage_address,rd->code_length);
if ((mode == kCommPage64) && (rd->special & kPort32to64))
commpage_port_32_to_64(rd->commpage_address,rd->code_length);
}
}
static void
commpage_populate_one(
vm_map_t submap, char ** kernAddressPtr, int mode, const char* signature ) {
char c1;
short c2;
addr64_t c8;
static double two52 = 1048576.0 * 1048576.0 * 4096.0; static double ten6 = 1000000.0; static uint64_t magicFE = 0xFEFEFEFEFEFEFEFFLL; static uint64_t magic80 = 0x8080808080808080LL; commpage_descriptor **rd;
short version = _COMM_PAGE_THIS_VERSION;
next = NULL; cur_routine = 0;
commPagePtr = (char*) commpage_allocate( submap );
*kernAddressPtr = commPagePtr;
commpage_stuff(_COMM_PAGE_SIGNATURE,signature,strlen(signature));
commpage_stuff(_COMM_PAGE_VERSION,&version,2);
commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int));
c1 = (_cpu_capabilities & kHasAltivec) ? -1 : 0;
commpage_stuff(_COMM_PAGE_ALTIVEC,&c1,1);
c1 = (_cpu_capabilities & k64Bit) ? -1 : 0;
commpage_stuff(_COMM_PAGE_64_BIT,&c1,1);
if (_cpu_capabilities & kCache32)
c2 = 32;
else if (_cpu_capabilities & kCache64)
c2 = 64;
else if (_cpu_capabilities & kCache128)
c2 = 128;
commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
commpage_stuff(_COMM_PAGE_2_TO_52,&two52,8);
commpage_stuff(_COMM_PAGE_10_TO_6,&ten6,8);
commpage_stuff(_COMM_PAGE_MAGIC_FE,&magicFE,8);
commpage_stuff(_COMM_PAGE_MAGIC_80,&magic80,8);
c8 = 0; commpage_stuff(_COMM_PAGE_TIMEBASE,&c8,8);
commpage_stuff(_COMM_PAGE_TIMESTAMP,&c8,8);
commpage_stuff(_COMM_PAGE_SEC_PER_TICK,&c8,8);
for( rd = routines; *rd != NULL ; rd++ )
commpage_stuff_routine(*rd,mode);
if (!matched)
panic("commpage no match on last routine");
if (next > (commPagePtr + _COMM_PAGE_AREA_USED))
panic("commpage overflow");
sync_cache_virtual((vm_offset_t) commPagePtr,_COMM_PAGE_AREA_USED);
}
void
commpage_populate( void )
{
commpage_init_cpu_capabilities();
commpage_populate_one( com_region_map32, &commPagePtr32, kCommPage32, "commpage 32-bit");
if (_cpu_capabilities & k64Bit) {
commpage_populate_one( com_region_map64, &commPagePtr64, kCommPage64, "commpage 64-bit");
pmap_init_sharedpage((vm_offset_t)commPagePtr64); }
}