diff --git a/platform/pc/lapic.c b/platform/pc/lapic.c index bb38ed8b..a5ab505f 100644 --- a/platform/pc/lapic.c +++ b/platform/pc/lapic.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,13 @@ static bool lapic_present = false; static bool lapic_x2apic = false; +static bool use_tsc_deadline = false; static volatile uint32_t *lapic_mmio; +static struct fp_32_64 timebase_to_lapic; + +// TODO: move these callbacks into the shared timer code +static platform_timer_callback t_callback; +static void *callback_arg; // local apic registers enum lapic_regs { @@ -83,7 +90,6 @@ enum lapic_timer_mode { LAPIC_TIMER_MODE_TSC_DEADLINE = 2, }; - static uint32_t lapic_read(enum lapic_regs reg) { LTRACEF("reg %#x\n", reg); DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI); @@ -116,33 +122,60 @@ static void lapic_write_icr(uint32_t low, uint32_t apic_id) { } } -void lapic_set_oneshot_timer(uint32_t tick) { - LTRACEF("tick %u\n", tick); +status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval) { + LTRACEF("tick %u\n", interval); - // set the initial count, which should trigger the timer - lapic_write(LAPIC_TICR, tick); + t_callback = callback; + callback_arg = arg; + + if (use_tsc_deadline) { + uint64_t now = __builtin_ia32_rdtsc(); + uint64_t delta = time_to_tsc_ticks(interval); + uint64_t deadline = now + delta; + LTRACEF("now %llu delta %llu deadline %llu\n", now, delta, deadline); + write_msr(X86_MSR_IA32_TSC_DEADLINE, deadline); + } else { + // set the initial count, which should trigger the timer + uint64_t ticks = u64_mul_u32_fp32_64(interval, timebase_to_lapic); + if (ticks > UINT32_MAX) { + ticks = UINT32_MAX; + } + + lapic_write(LAPIC_TICR, ticks & 0xffffffff); + } + + return NO_ERROR; } -void lapic_cancel_oneshot_timer(void) { +void lapic_cancel_timer(void) { LTRACE; - // set the counter to 0 which disables it - lapic_write(LAPIC_TICR, 0); + if (use_tsc_deadline) { + write_msr(X86_MSR_IA32_TSC_DEADLINE, 0); + } else { + lapic_write(LAPIC_TICR, 0); + } } -enum handler_return lapic_timer_handler(void *arg) { - //PANIC_UNIMPLEMENTED; +static enum handler_return lapic_timer_handler(void *arg) { + LTRACE; -// return timer_tick(NULL, current_time()); + enum handler_return ret = INT_NO_RESCHEDULE; + if (t_callback) { + ret = t_callback(callback_arg, current_time()); + } - lapic_set_oneshot_timer(100000000); - - return INT_NO_RESCHEDULE; + return ret; } void lapic_init(void) { - if (!lapic_present) + lapic_present = x86_feature_test(X86_FEATURE_APIC); +} + +void lapic_init_postvm(uint level) { + if (!lapic_present) { return; + } dprintf(INFO, "X86: local apic detected\n"); @@ -169,7 +202,7 @@ void lapic_init(void) { // map the lapic into the kernel since it's not guaranteed that the physmap covers it if (!lapic_mmio) { - dprintf(INFO, "X86: mapping lapic into kernel\n"); + LTRACEF("mapping lapic into kernel\n"); status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "lapic", PAGE_SIZE, (void **)&lapic_mmio, 0, apic_base & ~0xfff, /* vmm_flags */ 0, ARCH_MMU_FLAG_UNCACHED_DEVICE); ASSERT(err == NO_ERROR); @@ -186,17 +219,71 @@ void lapic_init(void) { if (eas) { dprintf(INFO, "X86: local apic EAS features %#x\n", lapic_read(LAPIC_EXT_FEATURES)); } +} +LK_INIT_HOOK(lapic_init_postvm, lapic_init_postvm, LK_INIT_LEVEL_VM + 1); - lapic_cancel_oneshot_timer(); +static uint32_t lapic_read_current_tick(void) { + if (!lapic_present) { + return 0; + } - // configure the local timer and make sure it is not set to fire - uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER; - lapic_write(LAPIC_TIMER, val); + return lapic_read(LAPIC_TCCR); +} + +void lapic_timer_init_percpu(uint level) { + // check for deadline mode + if (use_tsc_deadline) { + // put the timer in TSC deadline and clear the match register + uint32_t val = (LAPIC_TIMER_MODE_TSC_DEADLINE << 17) | LAPIC_INT_TIMER; + lapic_write(LAPIC_TIMER, val); + write_msr(X86_MSR_IA32_TSC_DEADLINE, 0); + } else { + // configure the local timer and make sure it is not set to fire + uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER; + lapic_write(LAPIC_TIMER, val); + lapic_write(LAPIC_TICR, 0); + } + + // register the local apic interrupts + register_int_handler_msi(LAPIC_INT_TIMER, &lapic_timer_handler, NULL, false); +} + +LK_INIT_HOOK_FLAGS(lapic_timer_init_percpu, lapic_timer_init_percpu, LK_INIT_LEVEL_VM, LK_INIT_FLAG_SECONDARY_CPUS); + +status_t lapic_timer_init(bool invariant_tsc_supported) { + if (!lapic_present) { + return ERR_NOT_FOUND; + } + + lapic_cancel_timer(); + + // check for deadline mode + bool tsc_deadline = x86_feature_test(X86_FEATURE_TSC_DEADLINE); + if (invariant_tsc_supported && tsc_deadline) { + dprintf(INFO, "X86: local apic timer supports TSC deadline mode\n"); + use_tsc_deadline = true; + } else { + // configure the local timer and make sure it is not set to fire + uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER; + lapic_write(LAPIC_TIMER, val); + + // calibrate the timer frequency + lapic_write(LAPIC_TICR, 0xffffffff); // countdown from the max count + uint32_t lapic_hz = pit_calibrate_lapic(&lapic_read_current_tick); + lapic_write(LAPIC_TICR, 0); + printf("X86: local apic timer frequency %uHz\n", lapic_hz); + + fp_32_64_div_32_32(&timebase_to_lapic, lapic_hz, 1000); + dprintf(INFO, "X86: timebase to local apic timer ratio %u.%08u...\n", + timebase_to_lapic.l0, timebase_to_lapic.l32); + } + + lapic_timer_init_percpu(0); // register the local apic interrupts register_int_handler_msi(LAPIC_INT_TIMER, &lapic_timer_handler, NULL, false); - lapic_set_oneshot_timer(1000000); + return NO_ERROR; } void lapic_eoi(unsigned int vector) { diff --git a/platform/pc/pit.c b/platform/pc/pit.c index 6b6faa2f..0b57f5f2 100644 --- a/platform/pc/pit.c +++ b/platform/pc/pit.c @@ -24,7 +24,6 @@ #define LOCAL_TRACE 0 - // TODO: switch this logic to lib/fixed_point math static platform_timer_callback t_callback; @@ -146,10 +145,12 @@ static void set_pit_frequency(uint32_t frequency) { } void pit_init(void) { + // start the PIT at 1Khz in free-running mode to keep a time base timer_current_time = 0; ticks_per_ms = INTERNAL_FREQ/1000; set_pit_frequency(1000); // ~1ms granularity register_int_handler(INT_PIT, &pit_timer_tick, NULL); + unmask_interrupt(INT_PIT); } status_t pit_set_periodic_timer(platform_timer_callback callback, void *arg, lk_time_t interval) { @@ -188,13 +189,32 @@ status_t pit_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_t return NO_ERROR; } +void pit_cancel_timer(void) { + LTRACE; + + spin_lock_saved_state_t state; + spin_lock_irqsave(&lock, state); + + next_trigger_time = 0; + + spin_unlock_irqrestore(&lock, state); +} + void pit_stop_timer(void) { LTRACE; spin_lock_saved_state_t state; spin_lock_irqsave(&lock, state); + next_trigger_time = 0; + next_trigger_delta = 0; + + // stop the PIT + outp(I8253_CONTROL_REG, 0x34); + outp(I8253_DATA_REG, 0); // LSB + outp(I8253_DATA_REG, 0); // MSB mask_interrupt(INT_PIT); + spin_unlock_irqrestore(&lock, state); } @@ -246,4 +266,53 @@ uint64_t pit_calibrate_tsc(void) { set_pit_frequency(1000); return tsc_freq; +} + +uint32_t pit_calibrate_lapic(uint32_t (*lapic_read_tick)(void)) { + DEBUG_ASSERT(arch_ints_disabled()); + + uint64_t lapic_ticks[5] = {0}; + uint32_t countdown_ms[5] = {0}; + + for (uint i = 0; i < countof(lapic_ticks); i++) { + // calibrate the tsc frequency using the PIT + countdown_ms[i] = 2 * (i + 1); + + uint16_t pic_ticks = INTERNAL_FREQ_TICKS_PER_MS * countdown_ms[i]; + outp(I8253_CONTROL_REG, 0x30); + outp(I8253_DATA_REG, pic_ticks & 0xff); // LSB + outp(I8253_DATA_REG, pic_ticks >> 8); // MSB + + // read the tsc + uint32_t tick_start = lapic_read_tick(); + + // wait for countdown_ms + uint8_t status = 0; + do { + // Send a read-back command that latches the status of ch0 + outp(I8253_CONTROL_REG, 0xe2); + status = inp(I8253_DATA_REG); + // Wait for bit 7 (output) to go high and for bit 6 (null count) to go low + } while ((status & 0xc0) != 0x80); + + uint32_t tick_end = lapic_read_tick(); + lapic_ticks[i] = tick_start - tick_end; + } + + // find the best time + uint best_index = 0; + for (uint i = 1; i < countof(lapic_ticks); i++) { + if (lapic_ticks[i] < lapic_ticks[best_index]) { + best_index = i; + } + } + + // calculate the tsc frequency + uint32_t freq = (lapic_ticks[best_index] * 1000) / countdown_ms[best_index]; + dprintf(INFO, "PIT: calibrated local apic frequency: %" PRIu32 "Hz\n", freq); + + // put the PIT back to 1ms countdown + set_pit_frequency(1000); + + return freq; } \ No newline at end of file diff --git a/platform/pc/platform_p.h b/platform/pc/platform_p.h index 59df1e0f..7e945aab 100644 --- a/platform/pc/platform_p.h +++ b/platform/pc/platform_p.h @@ -21,22 +21,30 @@ void pic_init(void); void pic_enable(unsigned int vector, bool enable); void pic_eoi(unsigned int vector); void pic_mask_interrupts(void); -uint64_t pit_calibrate_tsc(void); // local apic void lapic_init(void); +status_t lapic_timer_init(bool invariant_tsc_supported); void lapic_eoi(unsigned int vector); void lapic_send_init_ipi(uint32_t apic_id, bool level); void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector); void lapic_send_ipi(uint32_t apic_id, uint32_t vector); +status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval); +void lapic_cancel_timer(void); + +uint64_t time_to_tsc_ticks(lk_time_t time); + // programable interval timer void pit_init(void); status_t pit_set_periodic_timer(platform_timer_callback callback, void *arg, lk_time_t interval); status_t pit_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval); +void pit_cancel_timer(void); void pit_stop_timer(void); lk_time_t pit_current_time(void); lk_bigtime_t pit_current_time_hires(void); +uint64_t pit_calibrate_tsc(void); +uint32_t pit_calibrate_lapic(uint32_t (*lapic_read_tick)(void)); // secondary cpus void platform_start_secondary_cpus(void); diff --git a/platform/pc/timer.c b/platform/pc/timer.c index 03e77cdc..71460e22 100644 --- a/platform/pc/timer.c +++ b/platform/pc/timer.c @@ -22,7 +22,7 @@ #include #include -#define LOCAL_TRACE 1 +#define LOCAL_TRACE 0 // Deals with all of the various clock sources and event timers on the PC platform. @@ -33,8 +33,10 @@ static enum clock_source { CLOCK_SOURCE_HPET, } clock_source = CLOCK_SOURCE_INITIAL; -struct fp_32_64 tsc_to_timebase; -struct fp_32_64 tsc_to_timebase_hires; +static struct fp_32_64 tsc_to_timebase; +static struct fp_32_64 tsc_to_timebase_hires; +static struct fp_32_64 timebase_to_tsc; +static bool use_lapic_timer = false; static const char *clock_source_name(void) { switch (clock_source) { @@ -138,6 +140,11 @@ status_t pvclock_init(void) { return NO_ERROR; } +// Convert lk_time_t to TSC ticks +uint64_t time_to_tsc_ticks(lk_time_t time) { + return u64_mul_u32_fp32_64(time, timebase_to_tsc); +} + uint64_t pvclock_get_tsc_freq(void) { uint32_t tsc_mul = 0; int8_t tsc_shift = 0; @@ -182,26 +189,24 @@ void pc_init_timer(unsigned int level) { pit_init(); clock_source = CLOCK_SOURCE_PIT; - lapic_init(); - #if !X86_LEGACY // XXX update note about what invariant TSC means - bool invariant_tsc = x86_feature_test(X86_FEATURE_INVAR_TSC); - LTRACEF("invariant TSC %d\n", invariant_tsc); + bool use_invariant_tsc = x86_feature_test(X86_FEATURE_INVAR_TSC); + LTRACEF("invariant TSC %d\n", use_invariant_tsc); // Test for hypervisor PV clock, which also effectively says if TSC is invariant across // all cpus. if (pvclock_init() == NO_ERROR) { bool pv_clock_stable = pv_clock_is_stable(); - invariant_tsc |= pv_clock_stable; + use_invariant_tsc |= pv_clock_stable; printf("pv_clock: Clocksource is %sstable\n", (pv_clock_stable ? "" : "not ")); } // XXX test for HPET and use it over PIT if present - if (invariant_tsc) { + if (use_invariant_tsc) { // We're going to try to use the TSC as a time base, obtain the TSC frequency. uint64_t tsc_hz = 0; @@ -228,25 +233,51 @@ void pc_init_timer(unsigned int level) { dprintf(INFO, "PC: TSC to hires timebase ratio %u.%08u...\n", tsc_to_timebase_hires.l0, tsc_to_timebase_hires.l32); + fp_32_64_div_32_32(&timebase_to_tsc, tsc_hz, 1000); + dprintf(INFO, "PC: timebase to TSC ratio %u.%08u...\n", + timebase_to_tsc.l0, timebase_to_tsc.l32); + clock_source = CLOCK_SOURCE_TSC; } out: + + // Set up the local apic for event timer interrupts + if (lapic_timer_init(use_invariant_tsc) == NO_ERROR) { + dprintf(INFO, "PC: using LAPIC timer for event timer\n"); + use_lapic_timer = true; + } + + // If we're not using the PIT for time base and using the LAPIC timer for events, stop the PIT. + if (use_lapic_timer && clock_source != CLOCK_SOURCE_PIT) { + pit_stop_timer(); + } + #endif // !X86_LEGACY dprintf(INFO, "PC: using %s clock source\n", clock_source_name()); } -LK_INIT_HOOK(pc_timer, pc_init_timer, LK_INIT_LEVEL_VM); +LK_INIT_HOOK(pc_timer, pc_init_timer, LK_INIT_LEVEL_VM + 2); status_t platform_set_periodic_timer(platform_timer_callback callback, void *arg, lk_time_t interval) { + if (use_lapic_timer) { + PANIC_UNIMPLEMENTED; + } return pit_set_periodic_timer(callback, arg, interval); } status_t platform_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval) { + if (use_lapic_timer) { + return lapic_set_oneshot_timer(callback, arg, interval); + } return pit_set_oneshot_timer(callback, arg, interval); } void platform_stop_timer(void) { - pit_stop_timer(); + if (use_lapic_timer) { + lapic_cancel_timer(); + } else { + pit_cancel_timer(); + } }