#include <platforms.h>
#include <mach_kdb.h>
#include <mach/mach_types.h>
#include <kern/cpu_data.h>
#include <kern/cpu_number.h>
#include <kern/clock.h>
#include <kern/host_notify.h>
#include <kern/macro_help.h>
#include <kern/misc_protos.h>
#include <kern/spl.h>
#include <kern/assert.h>
#include <mach/vm_prot.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <i386/ipl.h>
#include <architecture/i386/pio.h>
#include <i386/misc_protos.h>
#include <i386/proc_reg.h>
#include <i386/machine_cpu.h>
#include <i386/lapic.h>
#include <i386/cpuid.h>
#include <i386/cpu_data.h>
#include <i386/cpu_threads.h>
#include <i386/perfmon.h>
#include <i386/machine_routines.h>
#include <pexpert/pexpert.h>
#include <machine/limits.h>
#include <machine/commpage.h>
#include <sys/kdebug.h>
#include <i386/tsc.h>
#include <i386/rtclock.h>
#define NSEC_PER_HZ (NSEC_PER_SEC / 100)
#define UI_CPUFREQ_ROUNDING_FACTOR 10000000
int rtclock_config(void);
int rtclock_init(void);
uint64_t rtc_decrementer_min;
void rtclock_intr(x86_saved_state_t *regs);
static uint64_t maxDec;
extern clock_timer_func_t rtclock_timer_expire;
static void rtc_set_timescale(uint64_t cycles);
static uint64_t rtc_export_speed(uint64_t cycles);
rtc_nanotime_t rtc_nanotime_info = {0,0,0,0,1,0};
static inline uint64_t
_tsc_to_nanoseconds(uint64_t value)
{
asm volatile("movl %%edx,%%esi ;"
"mull %%ecx ;"
"movl %%edx,%%edi ;"
"movl %%esi,%%eax ;"
"mull %%ecx ;"
"addl %%edi,%%eax ;"
"adcl $0,%%edx "
: "+A" (value)
: "c" (current_cpu_datap()->cpu_nanotime->scale)
: "esi", "edi");
return (value);
}
static uint32_t
deadline_to_decrementer(
uint64_t deadline,
uint64_t now)
{
uint64_t delta;
if (deadline <= now)
return rtc_decrementer_min;
else {
delta = deadline - now;
return MIN(MAX(rtc_decrementer_min,delta),maxDec);
}
}
void
rtc_lapic_start_ticking(void)
{
x86_lcpu_t *lcpu = x86_lcpu();
lcpu->rtcPop = EndOfAllTime;
etimer_resync_deadlines();
}
int
rtclock_config(void)
{
return (1);
}
static inline void
rtc_nanotime_set_commpage(rtc_nanotime_t *rntp)
{
commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift);
}
static inline void
_rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base)
{
uint64_t tsc = rdtsc64();
_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
}
static void
rtc_nanotime_init(uint64_t base)
{
rtc_nanotime_t *rntp = current_cpu_datap()->cpu_nanotime;
_rtc_nanotime_init(rntp, base);
rtc_nanotime_set_commpage(rntp);
}
void
rtc_nanotime_init_commpage(void)
{
spl_t s = splclock();
rtc_nanotime_set_commpage(current_cpu_datap()->cpu_nanotime);
splx(s);
}
static inline uint64_t
rtc_nanotime_read(void)
{
#if CONFIG_EMBEDDED
if (gPEClockFrequencyInfo.timebase_frequency_hz > SLOW_TSC_THRESHOLD)
return _rtc_nanotime_read(current_cpu_datap()->cpu_nanotime, 1);
else
#endif
return _rtc_nanotime_read(current_cpu_datap()->cpu_nanotime, 0);
}
void
rtc_clock_napped(uint64_t base, uint64_t tsc_base)
{
rtc_nanotime_t *rntp = current_cpu_datap()->cpu_nanotime;
uint64_t oldnsecs;
uint64_t newnsecs;
uint64_t tsc;
assert(!ml_get_interrupts_enabled());
tsc = rdtsc64();
oldnsecs = rntp->ns_base + _tsc_to_nanoseconds(tsc - rntp->tsc_base);
newnsecs = base + _tsc_to_nanoseconds(tsc - tsc_base);
if (oldnsecs < newnsecs) {
_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp);
rtc_nanotime_set_commpage(rntp);
}
}
void
rtc_clock_stepping(__unused uint32_t new_frequency,
__unused uint32_t old_frequency)
{
panic("rtc_clock_stepping unsupported");
}
void
rtc_clock_stepped(__unused uint32_t new_frequency,
__unused uint32_t old_frequency)
{
panic("rtc_clock_stepped unsupported");
}
void
rtc_sleep_wakeup(
uint64_t base)
{
rtc_nanotime_init(base);
}
int
rtclock_init(void)
{
uint64_t cycles;
assert(!ml_get_interrupts_enabled());
if (cpu_number() == master_cpu) {
assert(tscFreq);
rtc_set_timescale(tscFreq);
cycles = rtc_export_speed(tscFreq);
gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles;
gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles;
maxDec = tmrCvt(0x7fffffffULL, busFCvtt2n);
kprintf("maxDec: %lld\n", maxDec);
rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC, 0ULL);
lapic_set_timer_func((i386_intr_func_t) rtclock_intr);
clock_timebase_init();
ml_init_lock_timeout();
}
rtc_lapic_start_ticking();
return (1);
}
static void
rtc_set_timescale(uint64_t cycles)
{
rtc_nanotime_t *rntp = current_cpu_datap()->cpu_nanotime;
rntp->scale = ((uint64_t)NSEC_PER_SEC << 32) / cycles;
if (cycles <= SLOW_TSC_THRESHOLD)
rntp->shift = cycles;
else
rntp->shift = 32;
rtc_nanotime_init(0);
}
static uint64_t
rtc_export_speed(uint64_t cyc_per_sec)
{
uint64_t cycles;
cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2))
/ UI_CPUFREQ_ROUNDING_FACTOR)
* UI_CPUFREQ_ROUNDING_FACTOR;
if (cycles >= 0x100000000ULL) {
gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL;
} else {
gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles;
}
gPEClockFrequencyInfo.cpu_frequency_hz = cycles;
kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec);
return(cycles);
}
void
clock_get_system_microtime(
uint32_t *secs,
uint32_t *microsecs)
{
uint64_t now = rtc_nanotime_read();
uint32_t remain;
asm volatile(
"divl %3"
: "=a" (*secs), "=d" (remain)
: "A" (now), "r" (NSEC_PER_SEC));
asm volatile(
"divl %3"
: "=a" (*microsecs)
: "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
}
void
clock_get_system_nanotime(
uint32_t *secs,
uint32_t *nanosecs)
{
uint64_t now = rtc_nanotime_read();
asm volatile(
"divl %3"
: "=a" (*secs), "=d" (*nanosecs)
: "A" (now), "r" (NSEC_PER_SEC));
}
void
clock_gettimeofday_set_commpage(
uint64_t abstime,
uint64_t epoch,
uint64_t offset,
uint32_t *secs,
uint32_t *microsecs)
{
uint64_t now = abstime;
uint32_t remain;
now += offset;
asm volatile(
"divl %3"
: "=a" (*secs), "=d" (remain)
: "A" (now), "r" (NSEC_PER_SEC));
asm volatile(
"divl %3"
: "=a" (*microsecs)
: "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
*secs += epoch;
commpage_set_timestamp(abstime - remain, *secs);
}
void
clock_timebase_info(
mach_timebase_info_t info)
{
info->numer = info->denom = 1;
}
void
clock_set_timer_func(
clock_timer_func_t func)
{
if (rtclock_timer_expire == NULL)
rtclock_timer_expire = func;
}
void
rtclock_intr(
x86_saved_state_t *tregs)
{
uint64_t rip;
boolean_t user_mode = FALSE;
uint64_t abstime;
uint32_t latency;
x86_lcpu_t *lcpu = x86_lcpu();
assert(get_preemption_level() > 0);
assert(!ml_get_interrupts_enabled());
abstime = rtc_nanotime_read();
latency = (uint32_t)(abstime - lcpu->rtcDeadline);
if (abstime < lcpu->rtcDeadline)
latency = 1;
if (is_saved_state64(tregs) == TRUE) {
x86_saved_state64_t *regs;
regs = saved_state64(tregs);
user_mode = TRUE;
rip = regs->isf.rip;
} else {
x86_saved_state32_t *regs;
regs = saved_state32(tregs);
if (regs->cs & 0x03)
user_mode = TRUE;
rip = regs->eip;
}
KERNEL_DEBUG_CONSTANT(
MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
-latency, (uint32_t)rip, user_mode, 0, 0);
etimer_intr(user_mode, rip);
}
int
setPop(
uint64_t time)
{
uint64_t now;
uint32_t decr;
uint64_t count;
now = rtc_nanotime_read();
decr = deadline_to_decrementer(time, now);
count = tmrCvt(decr, busFCvtn2t);
lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count);
return decr;
}
uint64_t
mach_absolute_time(void)
{
return rtc_nanotime_read();
}
void
clock_interval_to_absolutetime_interval(
uint32_t interval,
uint32_t scale_factor,
uint64_t *result)
{
*result = (uint64_t)interval * scale_factor;
}
void
absolutetime_to_microtime(
uint64_t abstime,
uint32_t *secs,
uint32_t *microsecs)
{
uint32_t remain;
asm volatile(
"divl %3"
: "=a" (*secs), "=d" (remain)
: "A" (abstime), "r" (NSEC_PER_SEC));
asm volatile(
"divl %3"
: "=a" (*microsecs)
: "0" (remain), "d" (0), "r" (NSEC_PER_USEC));
}
void
absolutetime_to_nanotime(
uint64_t abstime,
uint32_t *secs,
uint32_t *nanosecs)
{
asm volatile(
"divl %3"
: "=a" (*secs), "=d" (*nanosecs)
: "A" (abstime), "r" (NSEC_PER_SEC));
}
void
nanotime_to_absolutetime(
uint32_t secs,
uint32_t nanosecs,
uint64_t *result)
{
*result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs;
}
void
absolutetime_to_nanoseconds(
uint64_t abstime,
uint64_t *result)
{
*result = abstime;
}
void
nanoseconds_to_absolutetime(
uint64_t nanoseconds,
uint64_t *result)
{
*result = nanoseconds;
}
void
machine_delay_until(
uint64_t deadline)
{
uint64_t now;
do {
cpu_pause();
now = mach_absolute_time();
} while (now < deadline);
}