/*
* Copyright (c) 2000-2010 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
/*
* Mach Operating System
* Copyright (c) 1991,1990 Carnegie Mellon University
* All Rights Reserved.
*
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
*
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
*
* Carnegie Mellon requests users of this software to return to
*
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
*
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
/*
*/
#include <i386/asm.h>
#include <assym.s>
#include <i386/eflags.h>
#include <i386/trap.h>
#include <i386/rtclock_asm.h>
#define _ARCH_I386_ASM_HELP_H_ /* Prevent inclusion of user header */
#include <mach/i386/syscall_sw.h>
#include <i386/postcode.h>
#include <i386/proc_reg.h>
#include <mach/exception_types.h>
/*
* Low-memory handlers.
*/
#define LO_ALLINTRS EXT(lo_allintrs32)
#define LO_ALLTRAPS EXT(lo_alltraps32)
#define LO_SYSENTER EXT(lo_sysenter32)
#define LO_UNIX_SCALL EXT(lo_unix_scall32)
#define LO_MACH_SCALL EXT(lo_mach_scall32)
#define LO_MDEP_SCALL EXT(lo_mdep_scall32)
#define HI_DATA(lo_addr) ( (EXT(lo_addr) - EXT(hi_remap_data)) + HIGH_IDT_BASE )
#define HI_TEXT(lo_text) ( (EXT(lo_text) - EXT(hi_remap_text)) + HIGH_MEM_BASE )
/*
* Interrupt descriptor table and code vectors for it.
*/
#define IDT_BASE_ENTRY(vec,seg,type) \
.data .word seg .byte type
#define IDT_BASE_ENTRY_INT(vec,seg,type) \
.data .word seg .byte type
#define IDT_BASE_ENTRY_TG(vec,seg,type) \
.data .word seg .byte type
#define IDT_ENTRY(vec,type) IDT_BASE_ENTRY(vec,KERNEL32_CS,type)
#define IDT_ENTRY_INT(vec,type) IDT_BASE_ENTRY_INT(vec,KERNEL32_CS,type)
/*
* No error code. Clear error code and push trap number.
*/
#define EXCEPTION(n,name) \
IDT_ENTRY(name,K_INTR_GATE);\
Entry(name) pushl $(n) movl $(LO_ALLTRAPS),%ebx
/*
* Interrupt from user. Clear error code and push trap number.
*/
#define EXCEP_USR(n,name) \
IDT_ENTRY(name,U_INTR_GATE);\
Entry(name) pushl $(n) movl $(LO_ALLTRAPS),%ebx
/*
* Special interrupt code.
*/
#define EXCEP_SPC(n,name) \
IDT_ENTRY(name,K_INTR_GATE)
/*
* Special interrupt code from user.
*/
#define EXCEP_SPC_USR(n,name) \
IDT_ENTRY(name,U_INTR_GATE)
/*
* Extra-special interrupt code. Note that no offset may be
* specified in a task gate descriptor, so name is ignored.
*/
/* Double-fault fatal handler */
#define DF_FATAL_TASK(n,name) \
IDT_BASE_ENTRY_TG(0,DF_TSS,K_TASK_GATE)
/* machine-check handler */
#define MC_FATAL_TASK(n,name) \
IDT_BASE_ENTRY_TG(0,MC_TSS,K_TASK_GATE)
/*
* Error code has been pushed. Push trap number.
*/
#define EXCEP_ERR(n,name) \
IDT_ENTRY(name,K_INTR_GATE) pushl $(n) movl $(LO_ALLTRAPS),%ebx
/*
* Interrupt.
*/
#define INTERRUPT(n) \
IDT_ENTRY_INT(L_ ## n,K_INTR_GATE) L_ ## n: pushl $(n) movl $(LO_ALLINTRS),%ebx
.data
.align 12
Entry(master_idt)
Entry(hi_remap_data)
.text
.align 12
Entry(hi_remap_text)
EXCEPTION(0x00,t_zero_div)
EXCEP_SPC(0x01,hi_debug)
INTERRUPT(0x02) /* NMI */
EXCEP_USR(0x03,t_int3)
EXCEP_USR(0x04,t_into)
EXCEP_USR(0x05,t_bounds)
EXCEPTION(0x06,t_invop)
EXCEPTION(0x07,t_nofpu)
DF_FATAL_TASK(0x08,df_task_start)
EXCEPTION(0x09,a_fpu_over)
EXCEPTION(0x0a,a_inv_tss)
EXCEP_SPC(0x0b,hi_segnp)
EXCEP_ERR(0x0c,t_stack_fault)
EXCEP_SPC(0x0d,hi_gen_prot)
EXCEP_SPC(0x0e,hi_page_fault)
EXCEPTION(0x0f,t_trap_0f)
EXCEPTION(0x10,t_fpu_err)
EXCEPTION(0x11,t_trap_11)
MC_FATAL_TASK(0x12,mc_task_start)
EXCEPTION(0x13,t_sse_err)
EXCEPTION(0x14,t_trap_14)
EXCEPTION(0x15,t_trap_15)
EXCEPTION(0x16,t_trap_16)
EXCEPTION(0x17,t_trap_17)
EXCEPTION(0x18,t_trap_18)
EXCEPTION(0x19,t_trap_19)
EXCEPTION(0x1a,t_trap_1a)
EXCEPTION(0x1b,t_trap_1b)
EXCEPTION(0x1c,t_trap_1c)
EXCEPTION(0x1d,t_trap_1d)
EXCEPTION(0x1e,t_trap_1e)
EXCEPTION(0x1f,t_trap_1f)
INTERRUPT(0x20)
INTERRUPT(0x21)
INTERRUPT(0x22)
INTERRUPT(0x23)
INTERRUPT(0x24)
INTERRUPT(0x25)
INTERRUPT(0x26)
INTERRUPT(0x27)
INTERRUPT(0x28)
INTERRUPT(0x29)
INTERRUPT(0x2a)
INTERRUPT(0x2b)
INTERRUPT(0x2c)
INTERRUPT(0x2d)
INTERRUPT(0x2e)
INTERRUPT(0x2f)
INTERRUPT(0x30)
INTERRUPT(0x31)
INTERRUPT(0x32)
INTERRUPT(0x33)
INTERRUPT(0x34)
INTERRUPT(0x35)
INTERRUPT(0x36)
INTERRUPT(0x37)
INTERRUPT(0x38)
INTERRUPT(0x39)
INTERRUPT(0x3a)
INTERRUPT(0x3b)
INTERRUPT(0x3c)
INTERRUPT(0x3d)
INTERRUPT(0x3e)
INTERRUPT(0x3f)
INTERRUPT(0x40)
INTERRUPT(0x41)
INTERRUPT(0x42)
INTERRUPT(0x43)
INTERRUPT(0x44)
INTERRUPT(0x45)
INTERRUPT(0x46)
INTERRUPT(0x47)
INTERRUPT(0x48)
INTERRUPT(0x49)
INTERRUPT(0x4a)
INTERRUPT(0x4b)
INTERRUPT(0x4c)
INTERRUPT(0x4d)
INTERRUPT(0x4e)
INTERRUPT(0x4f)
INTERRUPT(0x50)
INTERRUPT(0x51)
INTERRUPT(0x52)
INTERRUPT(0x53)
INTERRUPT(0x54)
INTERRUPT(0x55)
INTERRUPT(0x56)
INTERRUPT(0x57)
INTERRUPT(0x58)
INTERRUPT(0x59)
INTERRUPT(0x5a)
INTERRUPT(0x5b)
INTERRUPT(0x5c)
INTERRUPT(0x5d)
INTERRUPT(0x5e)
INTERRUPT(0x5f)
INTERRUPT(0x60)
INTERRUPT(0x61)
INTERRUPT(0x62)
INTERRUPT(0x63)
INTERRUPT(0x64)
INTERRUPT(0x65)
INTERRUPT(0x66)
INTERRUPT(0x67)
INTERRUPT(0x68)
INTERRUPT(0x69)
INTERRUPT(0x6a)
INTERRUPT(0x6b)
INTERRUPT(0x6c)
INTERRUPT(0x6d)
INTERRUPT(0x6e)
INTERRUPT(0x6f)
INTERRUPT(0x70)
INTERRUPT(0x71)
INTERRUPT(0x72)
INTERRUPT(0x73)
INTERRUPT(0x74)
INTERRUPT(0x75)
INTERRUPT(0x76)
INTERRUPT(0x77)
INTERRUPT(0x78)
INTERRUPT(0x79)
INTERRUPT(0x7a)
INTERRUPT(0x7b)
INTERRUPT(0x7c)
INTERRUPT(0x7d)
INTERRUPT(0x7e)
EXCEP_USR(0x7f, t_dtrace_ret)
EXCEP_SPC_USR(0x80,hi_unix_scall)
EXCEP_SPC_USR(0x81,hi_mach_scall)
EXCEP_SPC_USR(0x82,hi_mdep_scall)
INTERRUPT(0x83)
INTERRUPT(0x84)
INTERRUPT(0x85)
INTERRUPT(0x86)
INTERRUPT(0x87)
INTERRUPT(0x88)
INTERRUPT(0x89)
INTERRUPT(0x8a)
INTERRUPT(0x8b)
INTERRUPT(0x8c)
INTERRUPT(0x8d)
INTERRUPT(0x8e)
INTERRUPT(0x8f)
INTERRUPT(0x90)
INTERRUPT(0x91)
INTERRUPT(0x92)
INTERRUPT(0x93)
INTERRUPT(0x94)
INTERRUPT(0x95)
INTERRUPT(0x96)
INTERRUPT(0x97)
INTERRUPT(0x98)
INTERRUPT(0x99)
INTERRUPT(0x9a)
INTERRUPT(0x9b)
INTERRUPT(0x9c)
INTERRUPT(0x9d)
INTERRUPT(0x9e)
INTERRUPT(0x9f)
INTERRUPT(0xa0)
INTERRUPT(0xa1)
INTERRUPT(0xa2)
INTERRUPT(0xa3)
INTERRUPT(0xa4)
INTERRUPT(0xa5)
INTERRUPT(0xa6)
INTERRUPT(0xa7)
INTERRUPT(0xa8)
INTERRUPT(0xa9)
INTERRUPT(0xaa)
INTERRUPT(0xab)
INTERRUPT(0xac)
INTERRUPT(0xad)
INTERRUPT(0xae)
INTERRUPT(0xaf)
INTERRUPT(0xb0)
INTERRUPT(0xb1)
INTERRUPT(0xb2)
INTERRUPT(0xb3)
INTERRUPT(0xb4)
INTERRUPT(0xb5)
INTERRUPT(0xb6)
INTERRUPT(0xb7)
INTERRUPT(0xb8)
INTERRUPT(0xb9)
INTERRUPT(0xba)
INTERRUPT(0xbb)
INTERRUPT(0xbc)
INTERRUPT(0xbd)
INTERRUPT(0xbe)
INTERRUPT(0xbf)
INTERRUPT(0xc0)
INTERRUPT(0xc1)
INTERRUPT(0xc2)
INTERRUPT(0xc3)
INTERRUPT(0xc4)
INTERRUPT(0xc5)
INTERRUPT(0xc6)
INTERRUPT(0xc7)
INTERRUPT(0xc8)
INTERRUPT(0xc9)
INTERRUPT(0xca)
INTERRUPT(0xcb)
INTERRUPT(0xcc)
INTERRUPT(0xcd)
INTERRUPT(0xce)
INTERRUPT(0xcf)
INTERRUPT(0xd0)
INTERRUPT(0xd1)
INTERRUPT(0xd2)
INTERRUPT(0xd3)
INTERRUPT(0xd4)
INTERRUPT(0xd5)
INTERRUPT(0xd6)
INTERRUPT(0xd7)
INTERRUPT(0xd8)
INTERRUPT(0xd9)
INTERRUPT(0xda)
INTERRUPT(0xdb)
INTERRUPT(0xdc)
INTERRUPT(0xdd)
INTERRUPT(0xde)
INTERRUPT(0xdf)
INTERRUPT(0xe0)
INTERRUPT(0xe1)
INTERRUPT(0xe2)
INTERRUPT(0xe3)
INTERRUPT(0xe4)
INTERRUPT(0xe5)
INTERRUPT(0xe6)
INTERRUPT(0xe7)
INTERRUPT(0xe8)
INTERRUPT(0xe9)
INTERRUPT(0xea)
INTERRUPT(0xeb)
INTERRUPT(0xec)
INTERRUPT(0xed)
INTERRUPT(0xee)
INTERRUPT(0xef)
INTERRUPT(0xf0)
INTERRUPT(0xf1)
INTERRUPT(0xf2)
INTERRUPT(0xf3)
INTERRUPT(0xf4)
INTERRUPT(0xf5)
INTERRUPT(0xf6)
INTERRUPT(0xf7)
INTERRUPT(0xf8)
INTERRUPT(0xf9)
INTERRUPT(0xfa)
INTERRUPT(0xfb)
INTERRUPT(0xfc)
INTERRUPT(0xfd)
INTERRUPT(0xfe)
EXCEPTION(0xff,t_preempt)
.data
Entry(lo_kernel_cr3)
.long 0
.long 0
.text
/*
* Trap/interrupt entry points.
*
* All traps must create the following save area on the PCB "stack":
*
* gs
* fs
* es
* ds
* edi
* esi
* ebp
* cr2 if page fault - otherwise unused
* ebx
* edx
* ecx
* eax
* trap number
* error code
* eip
* cs
* eflags
* user esp - if from user
* user ss - if from user
*/
ret_to_kernel:
jmp *1f
1: .long HI_TEXT(hi_ret_to_kernel)
ret_to_user:
jmp *1f
1: .long HI_TEXT(hi_ret_to_user)
Entry(hi_ret_to_user)
movl %esp,%ebx
movl %gs:CPU_ACTIVE_THREAD,%ecx
subl TH_PCB_ISS(%ecx),%ebx
movl $(WINDOWS_CLEAN),TH_COPYIO_STATE(%ecx)
movl TH_PCB_IDS(%ecx),%eax /* get debug state struct */
cmpl $0,%eax /* is there a debug state */
je 1f /* branch if not */
movl DS_DR0(%eax), %ecx /* Load the 32 bit debug registers */
movl %ecx, %db0
movl DS_DR1(%eax), %ecx
movl %ecx, %db1
movl DS_DR2(%eax), %ecx
movl %ecx, %db2
movl DS_DR3(%eax), %ecx
movl %ecx, %db3
movl DS_DR7(%eax), %eax
1:
addl %gs:CPU_HI_ISS,%ebx /* rebase PCB save area to high addr */
movl %gs:CPU_TASK_CR3,%ecx
movl %ecx,%gs:CPU_ACTIVE_CR3
movl %ebx,%esp /* switch to hi based PCB stack */
movl %ecx,%cr3 /* switch to user's address space */
cmpl $0,%eax /* is dr7 set to something? */
je 2f /* branch if not */
movl %eax,%db7 /* Set dr7 */
2:
Entry(hi_ret_to_kernel)
popl %eax /* ignore flavor of saved state */
EXT(ret_popl_gs):
popl %gs /* restore segment registers */
EXT(ret_popl_fs):
popl %fs
EXT(ret_popl_es):
popl %es
EXT(ret_popl_ds):
popl %ds
popa /* restore general registers */
addl $8,%esp /* discard trap number and error code */
cmpl $(SYSENTER_CS),4(%esp) /* test for fast entry/exit */
je fast_exit
EXT(ret_iret):
iret /* return from interrupt */
fast_exit:
popl %edx /* user return eip */
popl %ecx /* pop and toss cs */
andl $(~EFL_IF),(%esp) /* clear intrs enabled, see sti below */
popf /* flags - carry denotes failure */
popl %ecx /* user return esp */
sti /* interrupts enabled after sysexit */
sysexit
Entry(hi_unix_scall)
pushl %eax /* save system call number */
pushl $0 /* clear trap number slot */
pusha /* save the general registers */
movl $(LO_UNIX_SCALL),%ebx
jmp enter_lohandler
Entry(hi_mach_scall)
pushl %eax /* save system call number */
pushl $0 /* clear trap number slot */
pusha /* save the general registers */
movl $(LO_MACH_SCALL),%ebx
jmp enter_lohandler
Entry(hi_mdep_scall)
pushl %eax /* save system call number */
pushl $0 /* clear trap number slot */
pusha /* save the general registers */
movl $(LO_MDEP_SCALL),%ebx
jmp enter_lohandler
/*
* sysenter entry point
* Requires user code to set up:
* edx: user instruction pointer (return address)
* ecx: user stack pointer
* on which is pushed stub ret addr and saved ebx
* Return to user-space is made using sysexit.
* Note: sysenter/sysexit cannot be used for calls returning a value in edx,
* or requiring ecx to be preserved.
*/
Entry(hi_sysenter)
movl (%esp), %esp /* switch from intr stack to pcb */
/*
* Push values on to the PCB stack
* to cons up the saved state.
*/
pushl $(USER_DS) /* ss */
pushl %ecx /* uesp */
pushf /* flags */
/*
* Clear, among others, the Nested Task (NT) flags bit */
pushl $0
popfl
pushl $(SYSENTER_CS) /* cs */
hi_sysenter_2:
pushl %edx /* eip */
pushl %eax /* err/eax - syscall code */
pushl $0 /* clear trap number slot */
pusha /* save the general registers */
orl $(EFL_IF),R32_EFLAGS-R32_EDI(%esp) /* (edi was last reg pushed) */
movl $(LO_SYSENTER),%ebx
enter_lohandler:
pushl %ds
pushl %es
pushl %fs
pushl %gs
pushl $(SS_32) /* 32-bit state flavor */
enter_lohandler1:
mov %ss,%eax
mov %eax,%ds
mov %eax,%fs
mov %eax,%es /* switch to kernel data seg */
mov $(CPU_DATA_GS),%eax
mov %eax,%gs
cld /* clear direction flag */
/*
* Switch to kernel's address space if necessary
*/
movl HI_DATA(lo_kernel_cr3),%ecx
movl %cr3,%eax
cmpl %eax,%ecx
je 1f
movl %ecx,%cr3
movl %ecx,%gs:CPU_ACTIVE_CR3
1:
testb $3,R32_CS(%esp)
jz 2f
movl %esp,%edx /* came from user mode */
xor %ebp, %ebp
subl %gs:CPU_HI_ISS,%edx
movl %gs:CPU_ACTIVE_THREAD,%ecx
addl TH_PCB_ISS(%ecx),%edx /* rebase the high stack to a low address */
movl %edx,%esp
cmpl $0, TH_PCB_IDS(%ecx) /* Is there a debug register state? */
je 2f
movl $0, %ecx /* If so, reset DR7 (the control) */
movl %ecx, %dr7
2:
movl R32_TRAPNO(%esp),%ecx // Get the interrupt vector
addl $1,%gs:hwIntCnt(,%ecx,4) // Bump the count
jmp *%ebx
/*
* Page fault traps save cr2.
*/
Entry(hi_page_fault)
pushl $(T_PAGE_FAULT) /* mark a page fault trap */
pusha /* save the general registers */
movl %cr2,%eax /* get the faulting address */
movl %eax,R32_CR2-R32_EDI(%esp)/* save in esp save slot */
movl $(LO_ALLTRAPS),%ebx
jmp enter_lohandler
/*
* Debug trap. Check for single-stepping across system call into
* kernel. If this is the case, taking the debug trap has turned
* off single-stepping - save the flags register with the trace
* bit set.
*/
Entry(hi_debug)
testb $3,4(%esp)
jnz hi_debug_trap
/* trap came from kernel mode */
cmpl $(HI_TEXT(hi_mach_scall)),(%esp)
jne 6f
addl $12,%esp /* remove eip/cs/eflags from debug_trap */
jmp EXT(hi_mach_scall) /* continue system call entry */
6:
cmpl $(HI_TEXT(hi_mdep_scall)),(%esp)
jne 5f
addl $12,%esp /* remove eip/cs/eflags from debug_trap */
jmp EXT(hi_mdep_scall) /* continue system call entry */
5:
cmpl $(HI_TEXT(hi_unix_scall)),(%esp)
jne 4f
addl $12,%esp /* remove eip/cs/eflags from debug_trap */
jmp EXT(hi_unix_scall) /* continue system call entry */
4:
cmpl $(HI_TEXT(hi_sysenter)),(%esp)
jne hi_debug_trap
/*
* eip/cs/flags have been pushed on intr stack
* We have to switch to pcb stack and copy eflags.
* Note: setting the cs selector to SYSENTER_TF_CS
* will cause the return to user path to take the iret path so
* that eflags (containing the trap bit) is set atomically.
* In unix_syscall this is tested so that we'll rewind the pc
* to account for with sysenter or int entry.
*/
addl $8,%esp /* remove eip/cs */
pushl %ecx /* save %ecx */
movl 8(%esp),%ecx /* top of intr stack -> pcb stack */
xchgl %ecx,%esp /* switch to pcb stack */
pushl $(USER_DS) /* ss */
pushl %ss:(%ecx) /* %ecx into uesp slot */
pushl %ss:4(%ecx) /* eflags */
movl %ss:(%ecx),%ecx /* restore %ecx */
pushl $(SYSENTER_TF_CS) /* cs - not SYSENTER_CS for iret path */
jmp hi_sysenter_2 /* continue sysenter entry */
hi_debug_trap:
pushl $0
pushl $(T_DEBUG) /* handle as user trap */
pusha /* save the general registers */
movl $(LO_ALLTRAPS),%ebx
jmp enter_lohandler
/*
* General protection or segment-not-present fault.
* Check for a GP/NP fault in the kernel_return
* sequence * esp-> 0: trap code (NP or GP)
* 4: segment number in error
* 8 eip
* 12 cs
* 16 eflags
* 20 old registers (trap is from kernel)
*/
Entry(hi_gen_prot)
pushl $(T_GENERAL_PROTECTION) /* indicate fault type */
jmp trap_check_kernel_exit /* check for kernel exit sequence */
Entry(hi_segnp)
pushl $(T_SEGMENT_NOT_PRESENT)
/* indicate fault type */
trap_check_kernel_exit:
testb $3,12(%esp)
jnz hi_take_trap
/* trap was from kernel mode, so */
/* check for the kernel exit sequence */
cmpl $(HI_TEXT(ret_iret)),8(%esp) /* on IRET? */
je fault_iret
cmpl $(HI_TEXT(ret_popl_ds)),8(%esp) /* popping DS? */
je fault_popl_ds
cmpl $(HI_TEXT(ret_popl_es)),8(%esp) /* popping ES? */
je fault_popl_es
cmpl $(HI_TEXT(ret_popl_fs)),8(%esp) /* popping FS? */
je fault_popl_fs
cmpl $(HI_TEXT(ret_popl_gs)),8(%esp) /* popping GS? */
je fault_popl_gs
hi_take_trap:
pusha /* save the general registers */
movl $(LO_ALLTRAPS),%ebx
jmp enter_lohandler
/*
* GP/NP fault on IRET: CS or SS is in error.
* All registers contain the user's values.
*
* on SP is
* 0 trap number
* 4 errcode
* 8 eip
* 12 cs --> trapno
* 16 efl --> errcode
* 20 user eip
* 24 user cs
* 28 user eflags
* 32 user esp
* 36 user ss
*/
fault_iret:
movl %eax,8(%esp) /* save eax (we don`t need saved eip) */
popl %eax /* get trap number */
movl %eax,12-4(%esp) /* put in user trap number */
popl %eax /* get error code */
movl %eax,16-8(%esp) /* put in user errcode */
popl %eax /* restore eax */
/* now treat as fault from user */
pusha /* save the general registers */
movl $(LO_ALLTRAPS),%ebx
jmp enter_lohandler
/*
* Fault restoring a segment register. The user's registers are still
* saved on the stack. The offending segment register has not been
* popped.
*/
fault_popl_ds:
popl %eax /* get trap number */
popl %edx /* get error code */
addl $12,%esp /* pop stack to user regs */
jmp push_es /* (DS on top of stack) */
fault_popl_es:
popl %eax /* get trap number */
popl %edx /* get error code */
addl $12,%esp /* pop stack to user regs */
jmp push_fs /* (ES on top of stack) */
fault_popl_fs:
popl %eax /* get trap number */
popl %edx /* get error code */
addl $12,%esp /* pop stack to user regs */
jmp push_gs /* (FS on top of stack) */
fault_popl_gs:
popl %eax /* get trap number */
popl %edx /* get error code */
addl $12,%esp /* pop stack to user regs */
jmp push_none /* (GS on top of stack) */
push_es:
pushl %es /* restore es, */
push_fs:
pushl %fs /* restore fs, */
push_gs:
pushl %gs /* restore gs. */
push_none:
pushl $(SS_32) /* 32-bit state flavor */
movl %eax,R32_TRAPNO(%esp) /* set trap number */
movl %edx,R32_ERR(%esp) /* set error code */
/* now treat as fault from user */
/* except that segment registers are */
/* already pushed */
movl $(LO_ALLTRAPS),%ebx
jmp enter_lohandler1
.text
Entry(hi_remap_etext)
/*
* All 32 bit task 'exceptions' enter lo_alltraps:
* esp -> x86_saved_state_t
*
* The rest of the state is set up as:
* cr3 -> kernel directory
* esp -> low based stack
* gs -> CPU_DATA_GS
* cs -> KERNEL32_CS
* ss/ds/es -> KERNEL_DS
*
* interrupts disabled
* direction flag cleared
*/
Entry(lo_alltraps32)
movl R32_CS(%esp),%eax /* assume 32-bit state */
cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */
jne 1f
movl R64_CS(%esp),%eax /* 64-bit user mode */
1:
testb $3,%al
jz trap_from_kernel
/* user mode trap */
TIME_TRAP_UENTRY
movl %gs:CPU_ACTIVE_THREAD,%ecx
movl TH_TASK(%ecx),%ebx
/* Check for active vtimers in the current task */
TASK_VTIMER_CHECK(%ebx, %ecx)
movl %gs:CPU_KERNEL_STACK,%ebx
xchgl %ebx,%esp /* switch to kernel stack */
CCALL1(user_trap, %ebx) /* call user trap routine */
/* user_trap() unmasks interrupts */
cli /* hold off intrs - critical section */
xorl %ecx,%ecx /* don't check if we're in the PFZ */
/*
* Return from trap or system call, checking for ASTs.
* On lowbase PCB stack with intrs disabled
*/
Entry(return_from_trap32)
movl %gs:CPU_ACTIVE_THREAD, %esp
movl TH_PCB_ISS(%esp), %esp /* switch back to PCB stack */
movl %gs:CPU_PENDING_AST, %eax
testl %eax, %eax
je EXT(return_to_user) /* branch if no AST */
LEXT(return_from_trap_with_ast)
movl %gs:CPU_KERNEL_STACK, %ebx
xchgl %ebx, %esp /* switch to kernel stack */
testl %ecx, %ecx /* see if we need to check for an EIP in the PFZ */
je 2f /* no, go handle the AST */
cmpl $(SS_64), SS_FLAVOR(%ebx) /* are we a 64-bit task? */
je 1f
/* no... 32-bit user mode */
movl R32_EIP(%ebx), %eax
pushl %ebx /* save PCB stack */
xorl %ebp, %ebp /* clear frame pointer */
CCALL1(commpage_is_in_pfz32, %eax)
popl %ebx /* retrieve pointer to PCB stack */
testl %eax, %eax
je 2f /* not in the PFZ... go service AST */
movl %eax, R32_EBX(%ebx) /* let the PFZ know we've pended an AST */
xchgl %ebx, %esp /* switch back to PCB stack */
jmp EXT(return_to_user)
1: /* 64-bit user mode */
movl R64_RIP(%ebx), %ecx
movl R64_RIP+4(%ebx), %eax
pushl %ebx /* save PCB stack */
xorl %ebp, %ebp /* clear frame pointer */
CCALL2(commpage_is_in_pfz64, %ecx, %eax)
popl %ebx /* retrieve pointer to PCB stack */
testl %eax, %eax
je 2f /* not in the PFZ... go service AST */
movl %eax, R64_RBX(%ebx) /* let the PFZ know we've pended an AST */
xchgl %ebx, %esp /* switch back to PCB stack */
jmp EXT(return_to_user)
2:
sti /* interrupts always enabled on return to user mode */
xorl %ebp, %ebp /* Clear framepointer */
CCALL1(i386_astintr, $0) /* take the AST */
cli
xorl %ecx, %ecx /* don't check if we're in the PFZ */
jmp EXT(return_from_trap32) /* and check again (rare) */
/*
* Trap from kernel mode. No need to switch stacks.
* Interrupts must be off here - we will set them to state at time of trap
* as soon as it's safe for us to do so and not recurse doing preemption
*/
trap_from_kernel:
movl %esp, %eax /* saved state addr */
pushl R32_EIP(%esp) /* Simulate a CALL from fault point */
pushl %ebp /* Extend framepointer chain */
movl %esp, %ebp
CCALL1WITHSP(kernel_trap, %eax) /* Call kernel trap handler */
popl %ebp
addl $4, %esp
cli
movl %gs:CPU_PENDING_AST,%eax /* get pending asts */
testl $ AST_URGENT,%eax /* any urgent preemption? */
je ret_to_kernel /* no, nothing to do */
cmpl $ T_PREEMPT,R32_TRAPNO(%esp)
je ret_to_kernel /* T_PREEMPT handled in kernel_trap() */
testl $ EFL_IF,R32_EFLAGS(%esp) /* interrupts disabled? */
je ret_to_kernel
cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
jne ret_to_kernel
movl %gs:CPU_KERNEL_STACK,%eax
movl %esp,%ecx
xorl %eax,%ecx
and EXT(kernel_stack_mask),%ecx
testl %ecx,%ecx /* are we on the kernel stack? */
jne ret_to_kernel /* no, skip it */
CCALL1(i386_astintr, $1) /* take the AST */
jmp ret_to_kernel
/*
* All interrupts on all tasks enter here with:
* esp-> -> x86_saved_state_t
*
* cr3 -> kernel directory
* esp -> low based stack
* gs -> CPU_DATA_GS
* cs -> KERNEL32_CS
* ss/ds/es -> KERNEL_DS
*
* interrupts disabled
* direction flag cleared
*/
Entry(lo_allintrs32)
/*
* test whether already on interrupt stack
*/
movl %gs:CPU_INT_STACK_TOP,%ecx
cmpl %esp,%ecx
jb 1f
leal -INTSTACK_SIZE(%ecx),%edx
cmpl %esp,%edx
jb int_from_intstack
1:
xchgl %ecx,%esp /* switch to interrupt stack */
movl %cr0,%eax /* get cr0 */
orl $(CR0_TS),%eax /* or in TS bit */
movl %eax,%cr0 /* set cr0 */
subl $8, %esp /* for 16-byte stack alignment */
pushl %ecx /* save pointer to old stack */
movl %ecx,%gs:CPU_INT_STATE /* save intr state */
TIME_INT_ENTRY /* do timing */
movl %gs:CPU_ACTIVE_THREAD,%ecx
movl TH_TASK(%ecx),%ebx
/* Check for active vtimers in the current task */
TASK_VTIMER_CHECK(%ebx, %ecx)
incl %gs:CPU_PREEMPTION_LEVEL
incl %gs:CPU_INTERRUPT_LEVEL
movl %gs:CPU_INT_STATE, %eax
CCALL1(interrupt, %eax) /* call generic interrupt routine */
cli /* just in case we returned with intrs enabled */
xorl %eax,%eax
movl %eax,%gs:CPU_INT_STATE /* clear intr state pointer */
decl %gs:CPU_INTERRUPT_LEVEL
decl %gs:CPU_PREEMPTION_LEVEL
TIME_INT_EXIT /* do timing */
movl %gs:CPU_ACTIVE_THREAD,%eax
movl TH_PCB_FPS(%eax),%eax /* get pcb's ifps */
testl %eax, %eax /* Is there a context */
je 1f /* Branch if not */
cmpl $0, FP_VALID(%eax) /* Check fp_valid */
jne 1f /* Branch if valid */
clts /* Clear TS */
jmp 2f
1:
movl %cr0,%eax /* get cr0 */
orl $(CR0_TS),%eax /* or in TS bit */
movl %eax,%cr0 /* set cr0 */
2:
popl %esp /* switch back to old stack */
/* Load interrupted code segment into %eax */
movl R32_CS(%esp),%eax /* assume 32-bit state */
cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */
jne 3f
movl R64_CS(%esp),%eax /* 64-bit user mode */
3:
testb $3,%al /* user mode, */
jnz ast_from_interrupt_user /* go handle potential ASTs */
/*
* we only want to handle preemption requests if
* the interrupt fell in the kernel context
* and preemption isn't disabled
*/
movl %gs:CPU_PENDING_AST,%eax
testl $ AST_URGENT,%eax /* any urgent requests? */
je ret_to_kernel /* no, nothing to do */
cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */
jne ret_to_kernel /* yes, skip it */
movl %gs:CPU_KERNEL_STACK,%eax
movl %esp,%ecx
xorl %eax,%ecx
and EXT(kernel_stack_mask),%ecx
testl %ecx,%ecx /* are we on the kernel stack? */
jne ret_to_kernel /* no, skip it */
/*
* Take an AST from kernel space. We don't need (and don't want)
* to do as much as the case where the interrupt came from user
* space.
*/
CCALL1(i386_astintr, $1)
jmp ret_to_kernel
/*
* nested int - simple path, can't preempt etc on way out
*/
int_from_intstack:
incl %gs:CPU_PREEMPTION_LEVEL
incl %gs:CPU_INTERRUPT_LEVEL
movl %esp, %edx /* x86_saved_state */
CCALL1(interrupt, %edx)
decl %gs:CPU_INTERRUPT_LEVEL
decl %gs:CPU_PREEMPTION_LEVEL
jmp ret_to_kernel
/*
* Take an AST from an interrupted user
*/
ast_from_interrupt_user:
movl %gs:CPU_PENDING_AST,%eax
testl %eax,%eax /* pending ASTs? */
je ret_to_user /* no, nothing to do */
TIME_TRAP_UENTRY
movl $1, %ecx /* check if we're in the PFZ */
jmp EXT(return_from_trap_with_ast) /* return */
/*
* 32bit Tasks
* System call entries via INTR_GATE or sysenter:
*
* esp -> x86_saved_state32_t
* cr3 -> kernel directory
* esp -> low based stack
* gs -> CPU_DATA_GS
* cs -> KERNEL32_CS
* ss/ds/es -> KERNEL_DS
*
* interrupts disabled
* direction flag cleared
*/
Entry(lo_sysenter32)
/*
* We can be here either for a mach syscall or a unix syscall,
* as indicated by the sign of the code:
*/
movl R32_EAX(%esp),%eax
testl %eax,%eax
js EXT(lo_mach_scall32) /* < 0 => mach */
/* > 0 => unix */
Entry(lo_unix_scall32)
TIME_TRAP_UENTRY
movl %gs:CPU_KERNEL_STACK,%edi
xchgl %edi,%esp /* switch to kernel stack */
movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
movl TH_TASK(%ecx),%ebx /* point to current task */
incl TH_SYSCALLS_UNIX(%ecx) /* increment call count */
/* Check for active vtimers in the current task */
TASK_VTIMER_CHECK(%ebx, %ecx)
sti
CCALL1(unix_syscall, %edi)
/*
* always returns through thread_exception_return
*/
Entry(lo_mach_scall32)
TIME_TRAP_UENTRY
movl %gs:CPU_KERNEL_STACK,%edi
xchgl %edi,%esp /* switch to kernel stack */
movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
movl TH_TASK(%ecx),%ebx /* point to current task */
incl TH_SYSCALLS_MACH(%ecx) /* increment call count */
/* Check for active vtimers in the current task */
TASK_VTIMER_CHECK(%ebx, %ecx)
sti
CCALL1(mach_call_munger, %edi)
/*
* always returns through thread_exception_return
*/
Entry(lo_mdep_scall32)
TIME_TRAP_UENTRY
movl %gs:CPU_KERNEL_STACK,%edi
xchgl %edi,%esp /* switch to kernel stack */
movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */
movl TH_TASK(%ecx),%ebx /* point to current task */
/* Check for active vtimers in the current task */
TASK_VTIMER_CHECK(%ebx, %ecx)
sti
CCALL1(machdep_syscall, %edi)
/*
* always returns through thread_exception_return
*/
LEXT(return_to_user)
TIME_TRAP_UEXIT
jmp ret_to_user
/*
* Double-fault exception handler task. The last gasp...
*/
Entry(df_task_start)
CCALL1(panic_double_fault32, $(T_DOUBLE_FAULT))
hlt
/*
* machine-check handler task. The last gasp...
*/
Entry(mc_task_start)
CCALL1(panic_machine_check32, $(T_MACHINE_CHECK))
hlt