#include <debug.h>
#include <types.h>
#include <mach/mach_types.h>
#include <mach/thread_status.h>
#include <kern/kern_types.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/misc_protos.h>
#include <kern/mach_param.h>
#include <kern/spl.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <ppc/misc_protos.h>
#include <ppc/cpu_internal.h>
#include <ppc/exception.h>
#include <ppc/proc_reg.h>
#include <ppc/pmap.h>
#include <ppc/trap.h>
#include <ppc/mappings.h>
#include <ppc/savearea.h>
#include <ppc/Firmware.h>
#include <ppc/asm.h>
#include <ppc/thread.h>
#include <ppc/vmachmon.h>
#include <ppc/low_trace.h>
#include <ppc/lowglobals.h>
#include <sys/kdebug.h>
void machine_act_terminate(thread_t);
#define KF_SIZE (FM_SIZE+ARG_SIZE+FM_REDZONE)
#if DEBUG
int fpu_trap_count = 0;
int fpu_switch_count = 0;
int vec_trap_count = 0;
int vec_switch_count = 0;
#endif
void
consider_machine_collect()
{
return;
}
void
consider_machine_adjust()
{
consider_mapping_adjust();
}
thread_t
machine_switch_context(
thread_t old,
thread_continue_t continuation,
thread_t new)
{
register thread_t retval;
pmap_t new_pmap;
facility_context *fowner;
struct per_proc_info *ppinfo;
if (old == new)
panic("machine_switch_context");
ppinfo = getPerProc();
ppinfo->old_thread = (unsigned int)old;
ppinfo->cpu_flags &= ~traceBE;
if(real_ncpus > 1) {
fowner = ppinfo->FPU_owner;
if(fowner) {
if(fowner->facAct == old) {
fpu_save(fowner);
}
}
fowner = ppinfo->VMX_owner;
if(fowner) {
if(fowner->facAct == old) {
vec_save(fowner);
}
}
}
if(old->machine.specFlags & runningVM) {
old->machine.specFlags &= ~(userProtKey|FamVMmode);
old->machine.specFlags |= (ppinfo->spcFlags) & (userProtKey|FamVMmode);
}
old->machine.specFlags &= ~OnProc;
new->machine.specFlags |= OnProc;
if(new->machine.specFlags & runningVM) {
pmap_switch(new->machine.vmmCEntry->vmmPmap);
ppinfo->VMMareaPhys = new->machine.vmmCEntry->vmmContextPhys;
ppinfo->VMMXAFlgs = new->machine.vmmCEntry->vmmXAFlgs;
ppinfo->FAMintercept = new->machine.vmmCEntry->vmmFAMintercept;
}
else {
new_pmap = new->task->map->pmap;
if ((old->task->map->pmap != new_pmap) || (old->machine.specFlags & runningVM)) {
pmap_switch(new_pmap);
}
}
if(old->machine.umwSpace != invalSpace) {
old->machine.umwSpace |= umwSwitchAway;
hw_blow_seg(lowGlo.lgUMWvaddr);
hw_blow_seg(lowGlo.lgUMWvaddr + 0x10000000ULL);
}
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
old->reason, (int)new, old->sched_pri, new->sched_pri, 0);
retval = Switch_context(old, continuation, new);
assert(retval != NULL);
if (branch_tracing_enabled()) {
ppinfo = getPerProc();
ppinfo->cpu_flags |= traceBE;
}
return retval;
}
kern_return_t
machine_thread_create(
thread_t thread,
task_t task)
{
savearea *sv;
unsigned int *CIsTooLimited, i;
hw_atomic_add((uint32_t *)&saveanchor.savetarget, 4);
assert(thread->machine.pcb == (savearea *)0);
sv = save_alloc();
bzero((char *)((unsigned int)sv + sizeof(savearea_comm)), (sizeof(savearea) - sizeof(savearea_comm)));
sv->save_hdr.save_prev = 0;
sv->save_hdr.save_flags = (sv->save_hdr.save_flags & ~SAVtype) | (SAVgeneral << SAVtypeshft);
sv->save_hdr.save_act = thread;
thread->machine.pcb = sv;
thread->machine.curctx = &thread->machine.facctx;
thread->machine.facctx.facAct = thread;
thread->machine.umwSpace = invalSpace;
thread->machine.preemption_count = 0;
thread->machine.upcb = sv;
sv->save_srr1 = (uint64_t)MSR_EXPORT_MASK_SET;
if(task_has_64BitAddr(task)) sv->save_srr1 |= (uint64_t)MASK32(MSR_SF) << 32;
sv->save_fpscr = 0;
sv->save_vrsave = 0;
sv->save_vscr[0] = 0x00000000;
sv->save_vscr[1] = 0x00000000;
sv->save_vscr[2] = 0x00000000;
sv->save_vscr[3] = 0x00010000;
return(KERN_SUCCESS);
}
void
machine_thread_destroy(
thread_t thread)
{
register savearea *pcb, *ppsv;
register savearea_vec *vsv, *vpsv;
register savearea_fpu *fsv, *fpsv;
register savearea *svp;
register int i;
boolean_t intr;
machine_act_terminate(thread);
intr = ml_set_interrupts_enabled(FALSE);
toss_live_vec(thread->machine.curctx);
vsv = thread->machine.curctx->VMXsave;
while(vsv) {
vpsv = vsv;
vsv = CAST_DOWN(savearea_vec *, vsv->save_hdr.save_prev);
save_release((savearea *)vpsv);
}
thread->machine.curctx->VMXsave = 0;
toss_live_fpu(thread->machine.curctx);
fsv = thread->machine.curctx->FPUsave;
while(fsv) {
fpsv = fsv;
fsv = CAST_DOWN(savearea_fpu *, fsv->save_hdr.save_prev);
save_release((savearea *)fpsv);
}
thread->machine.curctx->FPUsave = 0;
pcb = thread->machine.pcb;
while(pcb) {
ppsv = pcb;
pcb = CAST_DOWN(savearea *, pcb->save_hdr.save_prev);
save_release(ppsv);
}
hw_atomic_sub((uint32_t *)&saveanchor.savetarget, 4);
(void) ml_set_interrupts_enabled(intr);
}
void
act_machine_sv_free(thread_t act)
{
register savearea *pcb, *userpcb;
register savearea_vec *vsv, *vpst, *vsvt;
register savearea_fpu *fsv, *fpst, *fsvt;
register savearea *svp;
register int i;
boolean_t intr;
intr = ml_set_interrupts_enabled(FALSE);
if(act->machine.curctx->VMXlevel) {
toss_live_vec(act->machine.curctx);
if(!hw_lock_to((hw_lock_t)&act->machine.curctx->VMXsync, LockTimeOut)) {
panic("act_machine_sv_free - timeout getting VMX sync lock\n");
}
vsv = act->machine.curctx->VMXsave;
while(vsv && vsv->save_hdr.save_level) vsv = (savearea_vec *)vsv->save_hdr.save_prev;
vsvt = act->machine.curctx->VMXsave;
act->machine.curctx->VMXsave = vsv;
act->machine.curctx->VMXlevel = 0;
hw_lock_unlock((hw_lock_t)&act->machine.curctx->VMXsync);
while(vsvt) {
if (vsvt == vsv) break;
vpst = vsvt;
vsvt = (savearea_vec *)vsvt->save_hdr.save_prev;
save_ret((savearea *)vpst);
}
}
if(act->machine.curctx->FPUlevel) {
toss_live_fpu(act->machine.curctx);
if(!hw_lock_to((hw_lock_t)&act->machine.curctx->FPUsync, LockTimeOut)) {
panic("act_machine_sv_free - timeout getting FPU sync lock\n");
}
fsv = act->machine.curctx->FPUsave;
while(fsv && fsv->save_hdr.save_level) fsv = (savearea_fpu *)fsv->save_hdr.save_prev;
fsvt = act->machine.curctx->FPUsave;
act->machine.curctx->FPUsave = fsv;
act->machine.curctx->FPUlevel = 0;
hw_lock_unlock((hw_lock_t)&act->machine.curctx->FPUsync);
while(fsvt) {
if (fsvt == fsv) break;
fpst = fsvt;
fsvt = (savearea_fpu *)fsvt->save_hdr.save_prev;
save_ret((savearea *)fpst);
}
}
pcb = act->machine.pcb;
userpcb = 0;
while(pcb) {
if (pcb->save_srr1 & MASK(MSR_PR)) {
userpcb = pcb;
break;
}
svp = pcb;
pcb = CAST_DOWN(savearea *, pcb->save_hdr.save_prev);
save_ret(svp);
}
act->machine.pcb = userpcb;
(void) ml_set_interrupts_enabled(intr);
}
void
machine_act_terminate(
thread_t act)
{
if(act->machine.bbDescAddr) {
disable_bluebox_internal(act);
}
if(act->machine.vmmControl) {
vmm_tear_down_all(act);
}
}
void
machine_thread_terminate_self(void)
{
machine_act_terminate(current_thread());
}
void
machine_thread_init(void)
{
#ifdef MACHINE_STACK
#if KERNEL_STACK_SIZE > PPC_PGBYTES
panic("KERNEL_STACK_SIZE can't be greater than PPC_PGBYTES\n");
#endif
#endif
}
#if MACH_ASSERT
void
dump_thread(thread_t th)
{
printf(" thread @ 0x%x:\n", th);
}
int
dump_act(thread_t thr_act)
{
if (!thr_act)
return(0);
printf("thread(0x%x)(%d): task=%x(%d)\n",
thr_act, thr_act->ref_count,
thr_act->task, thr_act->task ? thr_act->task->ref_count : 0);
printf("\tsusp=%x active=%x\n",
thr_act->suspend_count, thr_act->active);
return((int)thr_act);
}
#endif
user_addr_t
get_useraddr()
{
return(current_thread()->machine.upcb->save_srr0);
}
vm_offset_t
machine_stack_detach(
thread_t thread)
{
vm_offset_t stack;
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_DETACH),
thread, thread->priority,
thread->sched_pri, 0, 0);
act_machine_sv_free(thread);
stack = thread->kernel_stack;
thread->kernel_stack = 0;
return(stack);
}
void
machine_stack_attach(
thread_t thread,
vm_offset_t stack)
{
unsigned int *kss;
struct savearea *sv;
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_ATTACH),
thread, thread->priority,
thread->sched_pri, 0, 0);
assert(stack);
kss = (unsigned int *)STACK_IKS(stack);
thread->kernel_stack = stack;
sv = save_get();
sv->save_hdr.save_flags = (sv->save_hdr.save_flags & ~SAVtype) | (SAVgeneral << SAVtypeshft);
sv->save_hdr.save_act = thread;
sv->save_hdr.save_prev = (addr64_t)((uintptr_t)thread->machine.pcb);
thread->machine.pcb = sv;
sv->save_srr0 = (unsigned int)thread_continue;
sv->save_r1 = (vm_offset_t)((int)kss - KF_SIZE);
sv->save_srr1 = MSR_SUPERVISOR_INT_OFF;
sv->save_fpscr = 0;
sv->save_vrsave = 0;
sv->save_vscr[3] = 0x00010000;
*(CAST_DOWN(int *, sv->save_r1)) = 0;
thread->machine.ksp = 0;
}
void
machine_stack_handoff(
thread_t old,
thread_t new)
{
vm_offset_t stack;
pmap_t new_pmap;
facility_context *fowner;
mapping_t *mp;
struct per_proc_info *ppinfo;
assert(new);
assert(old);
if (old == new)
panic("machine_stack_handoff");
stack = machine_stack_detach(old);
new->kernel_stack = stack;
if (stack == old->reserved_stack) {
assert(new->reserved_stack);
old->reserved_stack = new->reserved_stack;
new->reserved_stack = stack;
}
ppinfo = getPerProc();
ppinfo->cpu_flags &= ~traceBE;
if(real_ncpus > 1) {
fowner = ppinfo->FPU_owner;
if(fowner) {
if(fowner->facAct == old) {
fpu_save(fowner);
}
}
fowner = ppinfo->VMX_owner;
if(fowner) {
if(fowner->facAct == old) {
vec_save(fowner);
}
}
}
if(old->machine.specFlags & runningVM) {
old->machine.specFlags &= ~(userProtKey|FamVMmode);
old->machine.specFlags |= (ppinfo->spcFlags) & (userProtKey|FamVMmode);
}
old->machine.specFlags &= ~OnProc;
new->machine.specFlags |= OnProc;
KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_HANDOFF) | DBG_FUNC_NONE,
old->reason, (int)new, old->sched_pri, new->sched_pri, 0);
if(new->machine.specFlags & runningVM) {
pmap_switch(new->machine.vmmCEntry->vmmPmap);
ppinfo->VMMareaPhys = new->machine.vmmCEntry->vmmContextPhys;
ppinfo->VMMXAFlgs = new->machine.vmmCEntry->vmmXAFlgs;
ppinfo->FAMintercept = new->machine.vmmCEntry->vmmFAMintercept;
}
else {
new_pmap = new->task->map->pmap;
if ((old->task->map->pmap != new_pmap) || (old->machine.specFlags & runningVM)) {
pmap_switch(new_pmap);
}
}
machine_set_current_thread(new);
ppinfo->Uassist = new->machine.cthread_self;
ppinfo->ppbbTaskEnv = new->machine.bbTaskEnv;
ppinfo->spcFlags = new->machine.specFlags;
old->machine.umwSpace |= umwSwitchAway;
mp = (mapping_t *)&ppinfo->ppUMWmp;
mp->mpSpace = invalSpace;
if (branch_tracing_enabled())
ppinfo->cpu_flags |= traceBE;
if(trcWork.traceMask) dbgTrace(0x9903, (unsigned int)old, (unsigned int)new, 0, 0);
return;
}
void
call_continuation(
thread_continue_t continuation,
void *parameter,
wait_result_t wresult)
{
thread_t self = current_thread();
unsigned int *kss;
vm_offset_t tsp;
assert(self->kernel_stack);
kss = (unsigned int *)STACK_IKS(self->kernel_stack);
assert(continuation);
tsp = (vm_offset_t)((int)kss - KF_SIZE);
assert(tsp);
*((int *)tsp) = 0;
Call_continuation(continuation, parameter, wresult, tsp);
}