#ifdef KERNEL_PRIVATE
#ifndef _I386_MP_H_
#define _I386_MP_H_
#include <i386/apic.h>
#include <i386/mp_events.h>
#define MAX_CPUS 32
#ifndef ASSEMBLER
#include <stdint.h>
#include <sys/cdefs.h>
#include <mach/boolean.h>
#include <mach/kern_return.h>
#include <mach/i386/thread_status.h>
#include <mach/vm_types.h>
#include <kern/lock.h>
__BEGIN_DECLS
extern kern_return_t intel_startCPU(int slot_num);
extern kern_return_t intel_startCPU_fast(int slot_num);
extern void i386_init_slave(void);
extern void i386_init_slave_fast(void);
extern void smp_init(void);
extern void cpu_interrupt(int cpu);
__END_DECLS
extern unsigned int real_ncpus;
extern unsigned int max_ncpus;
decl_simple_lock_data(extern,kdb_lock)
__BEGIN_DECLS
extern void console_init(void);
extern void *console_cpu_alloc(boolean_t boot_cpu);
extern void console_cpu_free(void *console_buf);
extern int kdb_cpu;
extern int kdb_debug;
extern int kdb_active[];
extern volatile boolean_t mp_kdp_trap;
extern volatile boolean_t force_immediate_debugger_NMI;
extern volatile boolean_t pmap_tlb_flush_timeout;
extern uint64_t LastDebuggerEntryAllowance;
extern void mp_kdp_enter(void);
extern void mp_kdp_exit(void);
extern boolean_t mp_recent_debugger_activity(void);
#if MACH_KDB
extern void mp_kdb_exit(void);
#endif
extern void mp_rendezvous(
void (*setup_func)(void *),
void (*action_func)(void *),
void (*teardown_func)(void *),
void *arg);
extern void mp_rendezvous_no_intrs(
void (*action_func)(void *),
void *arg);
extern void mp_rendezvous_break_lock(void);
extern void mp_broadcast(
void (*action_func)(void *),
void *arg);
#if MACH_KDP
typedef long (*kdp_x86_xcpu_func_t) (void *arg0, void *arg1, uint16_t lcpu);
extern long kdp_x86_xcpu_invoke(const uint16_t lcpu,
kdp_x86_xcpu_func_t func,
void *arg0, void *arg1);
typedef enum {KDP_XCPU_NONE = 0xffff, KDP_CURRENT_LCPU = 0xfffe} kdp_cpu_t;
#endif
typedef uint32_t cpu_t;
typedef uint32_t cpumask_t;
static inline cpumask_t
cpu_to_cpumask(cpu_t cpu)
{
return (cpu < 32) ? (1 << cpu) : 0;
}
#define CPUMASK_ALL 0xffffffff
#define CPUMASK_SELF cpu_to_cpumask(cpu_number())
#define CPUMASK_OTHERS (CPUMASK_ALL & ~CPUMASK_SELF)
extern cpu_t mp_cpus_call(
cpumask_t cpus,
mp_sync_t mode,
void (*action_func)(void *),
void *arg);
extern void PM_interrupt_register(void (*fn)(void));
extern void cpu_PM_interrupt(int cpu);
__END_DECLS
#if MP_DEBUG
typedef struct {
uint64_t time;
int cpu;
mp_event_t event;
} cpu_signal_event_t;
#define LOG_NENTRIES 100
typedef struct {
uint64_t count[MP_LAST];
int next_entry;
cpu_signal_event_t entry[LOG_NENTRIES];
} cpu_signal_event_log_t;
extern cpu_signal_event_log_t *cpu_signal[];
extern cpu_signal_event_log_t *cpu_handle[];
#define DBGLOG(log,_cpu,_event) { \
boolean_t spl = ml_set_interrupts_enabled(FALSE); \
cpu_signal_event_log_t *logp = log[cpu_number()]; \
int next = logp->next_entry; \
cpu_signal_event_t *eventp = &logp->entry[next]; \
\
logp->count[_event]++; \
\
eventp->time = rdtsc64(); \
eventp->cpu = _cpu; \
eventp->event = _event; \
if (next == (LOG_NENTRIES - 1)) \
logp->next_entry = 0; \
else \
logp->next_entry++; \
\
(void) ml_set_interrupts_enabled(spl); \
}
#define DBGLOG_CPU_INIT(cpu) { \
cpu_signal_event_log_t **sig_logpp = &cpu_signal[cpu]; \
cpu_signal_event_log_t **hdl_logpp = &cpu_handle[cpu]; \
\
if (*sig_logpp == NULL && \
kmem_alloc(kernel_map, \
(vm_offset_t *) sig_logpp, \
sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\
panic("DBGLOG_CPU_INIT cpu_signal allocation failed\n");\
bzero(*sig_logpp, sizeof(cpu_signal_event_log_t)); \
if (*hdl_logpp == NULL && \
kmem_alloc(kernel_map, \
(vm_offset_t *) hdl_logpp, \
sizeof(cpu_signal_event_log_t)) != KERN_SUCCESS)\
panic("DBGLOG_CPU_INIT cpu_handle allocation failed\n");\
bzero(*hdl_logpp, sizeof(cpu_signal_event_log_t)); \
}
#else
#define DBGLOG(log,_cpu,_event)
#define DBGLOG_CPU_INIT(cpu)
#endif
#endif
#ifdef ASSEMBLER
#define i_bit(bit, word) ((long)(*(word)) & (1L << (bit)))
#else
static inline long
i_bit_impl(long word, long bit) {
return word & 1L << bit;
}
#define i_bit(bit, word) i_bit_impl((long)(*(word)), bit)
#endif
#define MP_DEV_OP_MAX 4
#define MP_DEV_WAIT MP_DEV_OP_MAX
#define MP_DEV_OP_START 0
#define MP_DEV_OP_INTR 1
#define MP_DEV_OP_TIMEO 2
#define MP_DEV_OP_CALLB 3
#if MACH_RT
#if defined(__i386__)
#define _DISABLE_PREEMPTION \
incl %gs:CPU_PREEMPTION_LEVEL
#define _ENABLE_PREEMPTION \
decl %gs:CPU_PREEMPTION_LEVEL ; \
jne 9f ; \
pushl %eax ; \
pushl %ecx ; \
pushl %edx ; \
call EXT(kernel_preempt_check) ; \
popl %edx ; \
popl %ecx ; \
popl %eax ; \
9:
#define _ENABLE_PREEMPTION_NO_CHECK \
decl %gs:CPU_PREEMPTION_LEVEL
#elif defined(__x86_64__)
#define _DISABLE_PREEMPTION \
incl %gs:CPU_PREEMPTION_LEVEL
#define _ENABLE_PREEMPTION \
decl %gs:CPU_PREEMPTION_LEVEL ; \
jne 9f ; \
call EXT(kernel_preempt_check) ; \
9:
#define _ENABLE_PREEMPTION_NO_CHECK \
decl %gs:CPU_PREEMPTION_LEVEL
#else
#error Unsupported architecture
#endif
#if MACH_ASSERT && defined(__i386__)
#define DISABLE_PREEMPTION \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
call EXT(_disable_preemption); \
popl %edx; \
popl %ecx; \
popl %eax
#define ENABLE_PREEMPTION \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
call EXT(_enable_preemption); \
popl %edx; \
popl %ecx; \
popl %eax
#define ENABLE_PREEMPTION_NO_CHECK \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
call EXT(_enable_preemption_no_check); \
popl %edx; \
popl %ecx; \
popl %eax
#define MP_DISABLE_PREEMPTION \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
call EXT(_mp_disable_preemption); \
popl %edx; \
popl %ecx; \
popl %eax
#define MP_ENABLE_PREEMPTION \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
call EXT(_mp_enable_preemption); \
popl %edx; \
popl %ecx; \
popl %eax
#define MP_ENABLE_PREEMPTION_NO_CHECK \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
call EXT(_mp_enable_preemption_no_check); \
popl %edx; \
popl %ecx; \
popl %eax
#else
#define DISABLE_PREEMPTION _DISABLE_PREEMPTION
#define ENABLE_PREEMPTION _ENABLE_PREEMPTION
#define ENABLE_PREEMPTION_NO_CHECK _ENABLE_PREEMPTION_NO_CHECK
#define MP_DISABLE_PREEMPTION _DISABLE_PREEMPTION
#define MP_ENABLE_PREEMPTION _ENABLE_PREEMPTION
#define MP_ENABLE_PREEMPTION_NO_CHECK _ENABLE_PREEMPTION_NO_CHECK
#endif
#else
#define DISABLE_PREEMPTION
#define ENABLE_PREEMPTION
#define ENABLE_PREEMPTION_NO_CHECK
#define MP_DISABLE_PREEMPTION
#define MP_ENABLE_PREEMPTION
#define MP_ENABLE_PREEMPTION_NO_CHECK
#endif
#endif
#endif