#include <mach_debug.h>
#include <mach_ldebug.h>
#include <sys/kdebug.h>
#include <mach/kern_return.h>
#include <mach/thread_status.h>
#include <mach/vm_param.h>
#include <kern/counters.h>
#include <kern/mach_param.h>
#include <kern/processor.h>
#include <kern/cpu_data.h>
#include <kern/cpu_number.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/sched_prim.h>
#include <kern/misc_protos.h>
#include <kern/assert.h>
#include <kern/spl.h>
#include <kern/machine.h>
#include <ipc/ipc_port.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/pmap.h>
#include <vm/vm_protos.h>
#include <i386/commpage/commpage.h>
#include <i386/cpu_data.h>
#include <i386/cpu_number.h>
#include <i386/cpuid.h>
#include <i386/eflags.h>
#include <i386/proc_reg.h>
#include <i386/tss.h>
#include <i386/user_ldt.h>
#include <i386/fpu.h>
#include <i386/mp_desc.h>
#include <i386/misc_protos.h>
#include <i386/thread.h>
#include <i386/seg.h>
#include <i386/machine_routines.h>
#if HYPERVISOR
#include <kern/hv_support.h>
#endif
#define ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(_type_) \
extern char assert_is_16byte_multiple_sizeof_ ## _type_ \
[(sizeof(_type_) % 16) == 0 ? 1 : -1]
ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_64_intr_stack_frame_t);
ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_saved_state_t);
#define DIRECTION_FLAG_DEBUG (DEBUG | DEVELOPMENT)
extern zone_t iss_zone;
extern zone_t ids_zone;
extern int tecs_mode_supported;
extern boolean_t cpuid_tsx_supported;
bool lbr_need_tsx_workaround = false;
int force_thread_policy_tecs;
struct lbr_group {
uint32_t msr_from;
uint32_t msr_to;
uint32_t msr_info;
};
struct cpu_lbrs {
uint32_t lbr_count;
struct lbr_group msr_lbrs[X86_MAX_LBRS];
};
const struct cpu_lbrs *cpu_lbr_setp = NULL;
int cpu_lbr_type;
const struct cpu_lbrs nhm_cpu_lbrs = {
16 ,
{
{ 0x680 , 0x6c0 , 0 },
{ 0x681 , 0x6c1 , 0 },
{ 0x682 , 0x6c2 , 0 },
{ 0x683 , 0x6c3 , 0 },
{ 0x684 , 0x6c4 , 0 },
{ 0x685 , 0x6c5 , 0 },
{ 0x686 , 0x6c6 , 0 },
{ 0x687 , 0x6c7 , 0 },
{ 0x688 , 0x6c8 , 0 },
{ 0x689 , 0x6c9 , 0 },
{ 0x68A , 0x6ca , 0 },
{ 0x68B , 0x6cb , 0 },
{ 0x68C , 0x6cc , 0 },
{ 0x68D , 0x6cd , 0 },
{ 0x68E , 0x6ce , 0 },
{ 0x68F , 0x6cf , 0 }
}
},
skl_cpu_lbrs = {
32 ,
{
{ 0x680 , 0x6c0 , 0xdc0 },
{ 0x681 , 0x6c1 , 0xdc1 },
{ 0x682 , 0x6c2 , 0xdc2 },
{ 0x683 , 0x6c3 , 0xdc3 },
{ 0x684 , 0x6c4 , 0xdc4 },
{ 0x685 , 0x6c5 , 0xdc5 },
{ 0x686 , 0x6c6 , 0xdc6 },
{ 0x687 , 0x6c7 , 0xdc7 },
{ 0x688 , 0x6c8 , 0xdc8 },
{ 0x689 , 0x6c9 , 0xdc9 },
{ 0x68A , 0x6ca , 0xdca },
{ 0x68B , 0x6cb , 0xdcb },
{ 0x68C , 0x6cc , 0xdcc },
{ 0x68D , 0x6cd , 0xdcd },
{ 0x68E , 0x6ce , 0xdce },
{ 0x68F , 0x6cf , 0xdcf },
{ 0x690 , 0x6d0 , 0xdd0 },
{ 0x691 , 0x6d1 , 0xdd1 },
{ 0x692 , 0x6d2 , 0xdd2 },
{ 0x693 , 0x6d3 , 0xdd3 },
{ 0x694 , 0x6d4 , 0xdd4 },
{ 0x695 , 0x6d5 , 0xdd5 },
{ 0x696 , 0x6d6 , 0xdd6 },
{ 0x697 , 0x6d7 , 0xdd7 },
{ 0x698 , 0x6d8 , 0xdd8 },
{ 0x699 , 0x6d9 , 0xdd9 },
{ 0x69a , 0x6da , 0xdda },
{ 0x69b , 0x6db , 0xddb },
{ 0x69c , 0x6dc , 0xddc },
{ 0x69d , 0x6dd , 0xddd },
{ 0x69e , 0x6de , 0xdde },
{ 0x69f , 0x6df , 0xddf }
}
};
void
i386_lbr_disable(void)
{
wrmsr64(MSR_IA32_DEBUGCTLMSR, rdmsr64(MSR_IA32_DEBUGCTLMSR) & ~DEBUGCTL_LBR_ENA);
}
void __attribute__((no_sanitize("address")))
i386_lbr_enable(void)
{
if (last_branch_support_enabled) {
wrmsr64(MSR_IA32_DEBUGCTLMSR, rdmsr64(MSR_IA32_DEBUGCTLMSR) | DEBUGCTL_LBR_ENA);
}
}
void __attribute__((no_sanitize("address")))
i386_lbr_init(i386_cpu_info_t *info_p, bool is_master)
{
if (!last_branch_support_enabled) {
i386_lbr_disable();
return;
}
if (is_master) {
cpu_lbr_type = PERFCAP_LBR_TYPE(rdmsr64(MSR_IA32_PERF_CAPABILITIES));
switch (info_p->cpuid_cpufamily) {
case CPUFAMILY_INTEL_NEHALEM:
case CPUFAMILY_INTEL_WESTMERE:
cpu_lbr_setp = &nhm_cpu_lbrs;
break;
case CPUFAMILY_INTEL_SANDYBRIDGE:
case CPUFAMILY_INTEL_IVYBRIDGE:
cpu_lbr_setp = &nhm_cpu_lbrs;
break;
case CPUFAMILY_INTEL_HASWELL:
case CPUFAMILY_INTEL_BROADWELL:
lbr_need_tsx_workaround = cpuid_tsx_supported ? false : true;
cpu_lbr_setp = &nhm_cpu_lbrs;
break;
case CPUFAMILY_INTEL_SKYLAKE:
case CPUFAMILY_INTEL_KABYLAKE:
case CPUFAMILY_INTEL_ICELAKE:
cpu_lbr_setp = &skl_cpu_lbrs;
break;
default:
panic("Unknown CPU family");
}
}
wrmsr64(MSR_IA32_LBR_SELECT, LBR_SELECT_CPL_EQ_0);
wrmsr64(MSR_IA32_DEBUGCTLMSR, rdmsr64(MSR_IA32_DEBUGCTLMSR) | DEBUGCTL_LBR_ENA);
}
int
i386_lbr_native_state_to_mach_thread_state(pcb_t pcb, last_branch_state_t *machlbrp)
{
int last_entry;
int i, j, lbr_tos;
uint64_t from_rip, to_rip;
#define LBR_SENTINEL_KERNEL_MODE (0x66726d6b65726e6cULL )
machlbrp->lbr_count = cpu_lbr_setp->lbr_count;
lbr_tos = pcb->lbrs.lbr_tos & (X86_MAX_LBRS - 1);
last_entry = (lbr_tos == (cpu_lbr_setp->lbr_count - 1)) ? 0 : (lbr_tos + 1);
switch (cpu_lbr_type) {
case PERFCAP_LBR_TYPE_MISPRED:
machlbrp->lbr_supported_tsx = 0;
machlbrp->lbr_supported_cycle_count = 0;
for (j = 0, i = lbr_tos;; (i = (i == 0) ? (cpu_lbr_setp->lbr_count - 1) : (i - 1)), j++) {
to_rip = pcb->lbrs.lbrs[i].to_rip;
machlbrp->lbrs[j].to_ip = (to_rip > VM_MAX_USER_PAGE_ADDRESS) ? LBR_SENTINEL_KERNEL_MODE : to_rip;
from_rip = LBR_TYPE_MISPRED_FROMRIP(pcb->lbrs.lbrs[i].from_rip);
machlbrp->lbrs[j].from_ip = (from_rip > VM_MAX_USER_PAGE_ADDRESS) ? LBR_SENTINEL_KERNEL_MODE : from_rip;
machlbrp->lbrs[j].mispredict = LBR_TYPE_MISPRED_MISPREDICT(pcb->lbrs.lbrs[i].from_rip);
machlbrp->lbrs[j].tsx_abort = machlbrp->lbrs[j].in_tsx = 0;
if (i == last_entry) {
break;
}
}
break;
case PERFCAP_LBR_TYPE_TSXINFO:
machlbrp->lbr_supported_tsx = cpuid_tsx_supported ? 1 : 0;
machlbrp->lbr_supported_cycle_count = 0;
for (j = 0, i = lbr_tos;; (i = (i == 0) ? (cpu_lbr_setp->lbr_count - 1) : (i - 1)), j++) {
to_rip = pcb->lbrs.lbrs[i].to_rip;
machlbrp->lbrs[j].to_ip = (to_rip > VM_MAX_USER_PAGE_ADDRESS) ? LBR_SENTINEL_KERNEL_MODE : to_rip;
from_rip = LBR_TYPE_TSXINFO_FROMRIP(pcb->lbrs.lbrs[i].from_rip);
machlbrp->lbrs[j].from_ip = (from_rip > VM_MAX_USER_PAGE_ADDRESS) ? LBR_SENTINEL_KERNEL_MODE : from_rip;
machlbrp->lbrs[j].mispredict = LBR_TYPE_TSXINFO_MISPREDICT(pcb->lbrs.lbrs[i].from_rip);
if (cpuid_tsx_supported) {
machlbrp->lbrs[j].tsx_abort = LBR_TYPE_TSXINFO_TSX_ABORT(pcb->lbrs.lbrs[i].from_rip);
machlbrp->lbrs[j].in_tsx = LBR_TYPE_TSXINFO_IN_TSX(pcb->lbrs.lbrs[i].from_rip);
} else {
machlbrp->lbrs[j].tsx_abort = 0;
machlbrp->lbrs[j].in_tsx = 0;
}
if (i == last_entry) {
break;
}
}
break;
case PERFCAP_LBR_TYPE_EIP_WITH_LBRINFO:
machlbrp->lbr_supported_tsx = cpuid_tsx_supported ? 1 : 0;
machlbrp->lbr_supported_cycle_count = 1;
for (j = 0, i = lbr_tos;; (i = (i == 0) ? (cpu_lbr_setp->lbr_count - 1) : (i - 1)), j++) {
from_rip = pcb->lbrs.lbrs[i].from_rip;
machlbrp->lbrs[j].from_ip = (from_rip > VM_MAX_USER_PAGE_ADDRESS) ? LBR_SENTINEL_KERNEL_MODE : from_rip;
to_rip = pcb->lbrs.lbrs[i].to_rip;
machlbrp->lbrs[j].to_ip = (to_rip > VM_MAX_USER_PAGE_ADDRESS) ? LBR_SENTINEL_KERNEL_MODE : to_rip;
machlbrp->lbrs[j].mispredict = LBR_TYPE_EIP_WITH_LBRINFO_MISPREDICT(pcb->lbrs.lbrs[i].info);
machlbrp->lbrs[j].tsx_abort = LBR_TYPE_EIP_WITH_LBRINFO_TSX_ABORT(pcb->lbrs.lbrs[i].info);
machlbrp->lbrs[j].in_tsx = LBR_TYPE_EIP_WITH_LBRINFO_IN_TSX(pcb->lbrs.lbrs[i].info);
machlbrp->lbrs[j].cycle_count = LBR_TYPE_EIP_WITH_LBRINFO_CYC_COUNT(pcb->lbrs.lbrs[i].info);
if (i == last_entry) {
break;
}
}
break;
default:
#if DEBUG || DEVELOPMENT
panic("Unknown LBR format: %d!", cpu_lbr_type);
#else
return -1;
#endif
}
return 0;
}
void
i386_lbr_synch(thread_t thr)
{
pcb_t old_pcb = THREAD_TO_PCB(thr);
int i;
if (cpu_lbr_setp->msr_lbrs[0].msr_info != 0) {
for (i = 0; i < cpu_lbr_setp->lbr_count; i++) {
old_pcb->lbrs.lbrs[i].from_rip = rdmsr64(cpu_lbr_setp->msr_lbrs[i].msr_from);
old_pcb->lbrs.lbrs[i].to_rip = rdmsr64(cpu_lbr_setp->msr_lbrs[i].msr_to);
old_pcb->lbrs.lbrs[i].info = rdmsr64(cpu_lbr_setp->msr_lbrs[i].msr_info);
}
} else {
for (i = 0; i < cpu_lbr_setp->lbr_count; i++) {
old_pcb->lbrs.lbrs[i].from_rip = rdmsr64(cpu_lbr_setp->msr_lbrs[i].msr_from);
old_pcb->lbrs.lbrs[i].to_rip = rdmsr64(cpu_lbr_setp->msr_lbrs[i].msr_to);
}
}
old_pcb->lbrs.lbr_tos = rdmsr64(MSR_IA32_LASTBRANCH_TOS);
}
void
i386_switch_lbrs(thread_t old, thread_t new)
{
pcb_t new_pcb;
int i;
bool save_old = (old != NULL && old->task != kernel_task);
bool restore_new = (new->task != kernel_task);
if (!save_old && !restore_new) {
return;
}
assert(cpu_lbr_setp != NULL);
new_pcb = THREAD_TO_PCB(new);
i386_lbr_disable();
if (save_old) {
i386_lbr_synch(old);
}
if (restore_new) {
if (cpu_lbr_setp->msr_lbrs[0].msr_info != 0) {
for (i = 0; i < cpu_lbr_setp->lbr_count; i++) {
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_from, new_pcb->lbrs.lbrs[i].from_rip);
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_to, new_pcb->lbrs.lbrs[i].to_rip);
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_info, new_pcb->lbrs.lbrs[i].info);
}
} else {
if (lbr_need_tsx_workaround) {
for (i = 0; i < cpu_lbr_setp->lbr_count; i++) {
#define BIT_47 (1ULL << 47)
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_from,
new_pcb->lbrs.lbrs[i].from_rip |
((new_pcb->lbrs.lbrs[i].from_rip & BIT_47) ? 0x6000000000000000ULL : 0));
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_to,
new_pcb->lbrs.lbrs[i].to_rip |
((new_pcb->lbrs.lbrs[i].to_rip & BIT_47) ? 0x6000000000000000ULL : 0));
}
} else {
for (i = 0; i < cpu_lbr_setp->lbr_count; i++) {
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_from, new_pcb->lbrs.lbrs[i].from_rip);
wrmsr64(cpu_lbr_setp->msr_lbrs[i].msr_to, new_pcb->lbrs.lbrs[i].to_rip);
}
}
}
wrmsr64(MSR_IA32_LASTBRANCH_TOS, new_pcb->lbrs.lbr_tos);
}
i386_lbr_enable();
}
void
act_machine_switch_pcb(thread_t old, thread_t new)
{
pcb_t pcb = THREAD_TO_PCB(new);
cpu_data_t *cdp = current_cpu_datap();
struct real_descriptor *ldtp;
mach_vm_offset_t pcb_stack_top;
assert(new->kernel_stack != 0);
assert(ml_get_interrupts_enabled() == FALSE);
#ifdef DIRECTION_FLAG_DEBUG
if (x86_get_flags() & EFL_DF) {
panic("Direction flag detected: 0x%lx", x86_get_flags());
}
#endif
set_ds(NULL_SEG);
set_es(NULL_SEG);
set_fs(NULL_SEG);
if (get_gs() != NULL_SEG) {
swapgs();
set_gs(NULL_SEG);
swapgs();
cdp->cpu_uber.cu_user_gs_base = 0;
}
vm_offset_t isf;
isf = (vm_offset_t) &pcb->iss->ss_64.isf;
cdp->cpu_uber.cu_isf = isf;
pcb_stack_top = (vm_offset_t) (pcb->iss + 1);
assert((pcb_stack_top & 0xF) == 0);
current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku;
*current_sstk64() = pcb_stack_top;
cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku;
if (is_saved_state64(pcb->iss)) {
cdp->cpu_task_map = new->map->pmap->pm_task_map;
gdt_desc_p(USER64_CS)->access |= ACC_PL_U;
gdt_desc_p(USER_CS)->access &= ~ACC_PL_U;
if ((pcb->cthread_self != 0) || (new->task != kernel_task)) {
if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) ||
(pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) {
cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
}
}
} else {
cdp->cpu_task_map = TASK_MAP_32BIT;
gdt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
gdt_desc_p(USER_CS)->access |= ACC_PL_U;
ldtp = current_ldt();
ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
if (pcb->uldt_selector != 0) {
ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
}
cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
}
cdp->cpu_curthread_do_segchk = new->machine.mthr_do_segchk;
if (last_branch_support_enabled) {
i386_switch_lbrs(old, new);
}
if (__probable(new->task == TASK_NULL || new->task->i386_ldt == 0)) {
ml_cpu_set_ldt(KERNEL_LDT);
cdp->cpu_curtask_has_ldt = 0;
} else {
user_ldt_set(new);
cdp->cpu_curtask_has_ldt = 1;
}
}
kern_return_t
thread_set_wq_state32(thread_t thread, thread_state_t tstate)
{
x86_thread_state32_t *state;
x86_saved_state32_t *saved_state;
thread_t curth = current_thread();
spl_t s = 0;
pal_register_cache_state(thread, DIRTY);
saved_state = USER_REGS32(thread);
state = (x86_thread_state32_t *)tstate;
if (curth != thread) {
s = splsched();
thread_lock(thread);
}
saved_state->ebp = 0;
saved_state->eip = state->eip;
saved_state->eax = state->eax;
saved_state->ebx = state->ebx;
saved_state->ecx = state->ecx;
saved_state->edx = state->edx;
saved_state->edi = state->edi;
saved_state->esi = state->esi;
saved_state->uesp = state->esp;
saved_state->efl = EFL_USER_SET;
saved_state->cs = USER_CS;
saved_state->ss = USER_DS;
saved_state->ds = USER_DS;
saved_state->es = USER_DS;
if (curth != thread) {
thread_unlock(thread);
splx(s);
}
return KERN_SUCCESS;
}
kern_return_t
thread_set_wq_state64(thread_t thread, thread_state_t tstate)
{
x86_thread_state64_t *state;
x86_saved_state64_t *saved_state;
thread_t curth = current_thread();
spl_t s = 0;
saved_state = USER_REGS64(thread);
state = (x86_thread_state64_t *)tstate;
if (!IS_USERADDR64_CANONICAL(state->rsp) ||
!IS_USERADDR64_CANONICAL(state->rip)) {
return KERN_FAILURE;
}
pal_register_cache_state(thread, DIRTY);
if (curth != thread) {
s = splsched();
thread_lock(thread);
}
saved_state->rbp = 0;
saved_state->rdi = state->rdi;
saved_state->rsi = state->rsi;
saved_state->rdx = state->rdx;
saved_state->rcx = state->rcx;
saved_state->r8 = state->r8;
saved_state->r9 = state->r9;
saved_state->isf.rip = state->rip;
saved_state->isf.rsp = state->rsp;
saved_state->isf.cs = USER64_CS;
saved_state->isf.rflags = EFL_USER_SET;
if (curth != thread) {
thread_unlock(thread);
splx(s);
}
return KERN_SUCCESS;
}
kern_return_t
machine_thread_create(
thread_t thread,
task_t task)
{
pcb_t pcb = THREAD_TO_PCB(thread);
if ((task->t_flags & TF_TECS) || __improbable(force_thread_policy_tecs)) {
thread->machine.mthr_do_segchk = 1;
} else {
thread->machine.mthr_do_segchk = 0;
}
if (pcb->iss == NULL) {
assert((get_preemption_level() == 0));
pcb->iss = (x86_saved_state_t *) zalloc(iss_zone);
if (pcb->iss == NULL) {
panic("iss_zone");
}
}
assert(sizeof(pcb->iss->ss_32) + sizeof(pcb->iss->ss_64.isf) <=
sizeof(pcb->iss->ss_64));
bzero((char *)pcb->iss, sizeof(x86_saved_state_t));
bzero(&pcb->lbrs, sizeof(x86_lbrs_t));
if (task_has_64Bit_addr(task)) {
pcb->iss->flavor = x86_SAVED_STATE64;
pcb->iss->ss_64.isf.cs = USER64_CS;
pcb->iss->ss_64.isf.ss = USER_DS;
pcb->iss->ss_64.fs = USER_DS;
pcb->iss->ss_64.gs = USER_DS;
pcb->iss->ss_64.isf.rflags = EFL_USER_SET;
} else {
pcb->iss->flavor = x86_SAVED_STATE32;
pcb->iss->ss_32.cs = USER_CS;
pcb->iss->ss_32.ss = USER_DS;
pcb->iss->ss_32.ds = USER_DS;
pcb->iss->ss_32.es = USER_DS;
pcb->iss->ss_32.fs = USER_DS;
pcb->iss->ss_32.gs = USER_DS;
pcb->iss->ss_32.efl = EFL_USER_SET;
}
simple_lock_init(&pcb->lock, 0);
pcb->cthread_self = 0;
pcb->uldt_selector = 0;
pcb->thread_gpu_ns = 0;
if ((pcb->cthread_desc.access & ACC_P) == 0) {
pcb->cthread_desc = *gdt_desc_p(USER_DS);
}
pcb->insn_state_copyin_failure_errorcode = 0;
if (pcb->insn_state != 0) {
bzero(pcb->insn_state, sizeof(x86_instruction_state_t));
pcb->insn_state->insn_stream_valid_bytes = -1;
}
return KERN_SUCCESS;
}
void
machine_thread_destroy(
thread_t thread)
{
pcb_t pcb = THREAD_TO_PCB(thread);
#if HYPERVISOR
if (thread->hv_thread_target) {
hv_callbacks.thread_destroy(thread->hv_thread_target);
thread->hv_thread_target = NULL;
}
#endif
if (pcb->ifps != 0) {
fpu_free(thread, pcb->ifps);
}
if (pcb->iss != 0) {
zfree(iss_zone, pcb->iss);
pcb->iss = 0;
}
if (pcb->ids) {
zfree(ids_zone, pcb->ids);
pcb->ids = NULL;
}
if (pcb->insn_state != 0) {
kfree(pcb->insn_state, sizeof(x86_instruction_state_t));
pcb->insn_state = 0;
}
pcb->insn_state_copyin_failure_errorcode = 0;
}
kern_return_t
machine_thread_set_tsd_base(
thread_t thread,
mach_vm_offset_t tsd_base)
{
if (thread->task == kernel_task) {
return KERN_INVALID_ARGUMENT;
}
if (thread_is_64bit_addr(thread)) {
if (!IS_USERADDR64_CANONICAL(tsd_base)) {
tsd_base = 0ULL;
}
} else {
if (tsd_base > UINT32_MAX) {
tsd_base = 0ULL;
}
}
pcb_t pcb = THREAD_TO_PCB(thread);
pcb->cthread_self = tsd_base;
if (!thread_is_64bit_addr(thread)) {
struct real_descriptor desc = {
.limit_low = 1,
.limit_high = 0,
.base_low = tsd_base & 0xffff,
.base_med = (tsd_base >> 16) & 0xff,
.base_high = (tsd_base >> 24) & 0xff,
.access = ACC_P | ACC_PL_U | ACC_DATA_W,
.granularity = SZ_32 | SZ_G,
};
pcb->cthread_desc = desc;
saved_state32(pcb->iss)->gs = USER_CTHREAD;
}
if (thread == current_thread()) {
if (thread_is_64bit_addr(thread)) {
cpu_data_t *cdp;
mp_disable_preemption();
cdp = current_cpu_datap();
if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) ||
(pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) {
wrmsr64(MSR_IA32_KERNEL_GS_BASE, tsd_base);
}
cdp->cpu_uber.cu_user_gs_base = tsd_base;
mp_enable_preemption();
} else {
mp_disable_preemption();
*ldt_desc_p(USER_CTHREAD) = pcb->cthread_desc;
mp_enable_preemption();
}
}
return KERN_SUCCESS;
}
void
machine_tecs(thread_t thr)
{
if (tecs_mode_supported) {
thr->machine.mthr_do_segchk = 1;
}
}
int
machine_csv(cpuvn_e cve)
{
switch (cve) {
case CPUVN_CI:
return (cpuid_wa_required(CPU_INTEL_SEGCHK) & CWA_ON) != 0;
default:
break;
}
return 0;
}