diff --git a/arch/x86/32/exceptions.S b/arch/x86/32/exceptions.S index 9e980e07..6cb06277 100644 --- a/arch/x86/32/exceptions.S +++ b/arch/x86/32/exceptions.S @@ -83,8 +83,6 @@ FUNCTION(setup_idt) loop .Lloop - lidt _idtr - ret END_FUNCTION(setup_idt) diff --git a/arch/x86/64/exceptions.S b/arch/x86/64/exceptions.S index c8f7ed9e..092056a1 100644 --- a/arch/x86/64/exceptions.S +++ b/arch/x86/64/exceptions.S @@ -113,8 +113,6 @@ FUNCTION(setup_idt) loop .Lloop - lidt _idtr - ret END_FUNCTION(setup_idt) diff --git a/arch/x86/arch.c b/arch/x86/arch.c index daa1dd3f..a20dc873 100644 --- a/arch/x86/arch.c +++ b/arch/x86/arch.c @@ -80,6 +80,9 @@ void x86_early_init_percpu(void) { x86_set_gdt_descriptor(selector, &system_tss, sizeof(system_tss), 1, 0, 0, SEG_TYPE_TSS, 0, 0); x86_ltr(selector); + /* load the kernel's IDT */ + asm("lidt _idtr"); + x86_mmu_early_init_percpu(); #if X86_WITH_FPU x86_fpu_early_init_percpu(); diff --git a/arch/x86/include/arch/x86/lapic.h b/arch/x86/include/arch/x86/lapic.h new file mode 100644 index 00000000..2fa421af --- /dev/null +++ b/arch/x86/include/arch/x86/lapic.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2025 Travis Geiselbrecht + * + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT + */ + +#pragma once + +#include +#include +#include +#include + +// local apic +void lapic_init(void); +status_t lapic_timer_init(bool invariant_tsc_supported); +void lapic_eoi(unsigned int vector); +void lapic_send_init_ipi(uint32_t apic_id, bool level); +void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector); +void lapic_send_ipi(uint32_t apic_id, mp_ipi_t ipi); + +status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval); +void lapic_cancel_timer(void); + diff --git a/arch/x86/include/arch/x86/mp.h b/arch/x86/include/arch/x86/mp.h index c668f4d0..8b4c6e0b 100644 --- a/arch/x86/include/arch/x86/mp.h +++ b/arch/x86/include/arch/x86/mp.h @@ -16,7 +16,7 @@ typedef struct x86_percpu { struct x86_percpu *self; uint cpu_num; - uint apic_id; + uint32_t apic_id; struct thread *current_thread; @@ -70,10 +70,13 @@ static inline uint x86_get_cpu_num(void) { } // get the current apic id -static inline uint x86_get_apic_id(void) { +static inline uint32_t x86_get_apic_id(void) { return x86_read_gs_offset32(X86_PERCPU_FIELD_OFFSET(apic_id)); } +// read it from hardware directly +uint32_t x86_get_apic_id_from_hardware(void); + // get/set the current thread struct thread; @@ -83,4 +86,4 @@ static inline struct thread *x86_get_current_thread(void) { static inline void x86_set_current_thread(struct thread *t) { x86_write_gs_offset_ptr(X86_PERCPU_FIELD_OFFSET(current_thread), t); -} \ No newline at end of file +} diff --git a/platform/pc/lapic.c b/arch/x86/lapic.c similarity index 78% rename from platform/pc/lapic.c rename to arch/x86/lapic.c index 36030e87..4b7e0793 100644 --- a/platform/pc/lapic.c +++ b/arch/x86/lapic.c @@ -5,6 +5,8 @@ * license that can be found in the LICENSE file or at * https://opensource.org/licenses/MIT */ +#include "arch/x86/lapic.h" + #include #include #include @@ -20,10 +22,10 @@ #include #include #include -#include +#include +#include #include - -#include "platform_p.h" +#include #define LOCAL_TRACE 0 @@ -37,7 +39,7 @@ static struct fp_32_64 timebase_to_lapic; static platform_timer_callback t_callback; static void *callback_arg; -static void lapic_timer_init_percpu(void); +static void lapic_init_percpu(uint level); // local apic registers enum lapic_regs { @@ -82,6 +84,7 @@ enum lapic_regs { enum lapic_interrupts { LAPIC_INT_TIMER = 0xf8, + LAPIC_INT_SPURIOUS, LAPIC_INT_GENERIC, LAPIC_INT_RESCHEDULE, }; @@ -93,7 +96,7 @@ enum lapic_timer_mode { }; static uint32_t lapic_read(enum lapic_regs reg) { - LTRACEF("reg %#x\n", reg); + LTRACEF_LEVEL(2, "reg %#x\n", reg); DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI); if (lapic_x2apic) { // TODO: do we need barriers here? @@ -104,7 +107,7 @@ static uint32_t lapic_read(enum lapic_regs reg) { } static void lapic_write(enum lapic_regs reg, uint32_t val) { - LTRACEF("reg %#x val %#x\n", reg, val); + LTRACEF_LEVEL(2, "reg %#x val %#x\n", reg, val); DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI); if (lapic_x2apic) { write_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10, val); @@ -115,7 +118,7 @@ static void lapic_write(enum lapic_regs reg, uint32_t val) { // special case to write to the ICR register static void lapic_write_icr(uint32_t low, uint32_t apic_id) { - LTRACEF("%#x apic_id %#x\n", low, apic_id); + LTRACEF_LEVEL(2, "%#x apic_id %#x\n", low, apic_id); if (lapic_x2apic) { write_msr(X86_MSR_IA32_X2APIC_BASE + 0x30, ((uint64_t)apic_id << 32) | low); } else { @@ -125,7 +128,9 @@ static void lapic_write_icr(uint32_t low, uint32_t apic_id) { } status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval) { - LTRACEF("tick %u\n", interval); + LTRACEF("cpu %u interval %u\n", arch_curr_cpu_num(), interval); + + DEBUG_ASSERT(arch_ints_disabled()); t_callback = callback; callback_arg = arg; @@ -152,6 +157,8 @@ status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk void lapic_cancel_timer(void) { LTRACE; + DEBUG_ASSERT(arch_ints_disabled()); + if (use_tsc_deadline) { write_msr(X86_MSR_IA32_TSC_DEADLINE, 0); } else { @@ -160,7 +167,7 @@ void lapic_cancel_timer(void) { } static enum handler_return lapic_timer_handler(void *arg) { - LTRACE; + LTRACEF("cpu %u\n", arch_curr_cpu_num()); enum handler_return ret = INT_NO_RESCHEDULE; if (t_callback) { @@ -170,11 +177,29 @@ static enum handler_return lapic_timer_handler(void *arg) { return ret; } +static enum handler_return lapic_spurious_handler(void *arg) { + LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg); + + return INT_NO_RESCHEDULE; +} + +static enum handler_return lapic_generic_handler(void *arg) { + LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg); + + return INT_NO_RESCHEDULE; +} + +static enum handler_return lapic_reschedule_handler(void *arg) { + LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg); + + return mp_mbx_reschedule_irq(); +} + void lapic_init(void) { lapic_present = x86_feature_test(X86_FEATURE_APIC); } -void lapic_init_postvm(uint level) { +static void lapic_init_postvm(uint level) { if (!lapic_present) { return; } @@ -221,10 +246,13 @@ void lapic_init_postvm(uint level) { if (eas) { dprintf(INFO, "X86: local apic EAS features %#x\n", lapic_read(LAPIC_EXT_FEATURES)); } + + // Finish up some local initialization that all cpus will want to do + lapic_init_percpu(0); } LK_INIT_HOOK(lapic_init_postvm, lapic_init_postvm, LK_INIT_LEVEL_VM + 1); -void lapic_init_percpu(uint level) { +static void lapic_init_percpu(uint level) { // Make sure the apic is enabled and x2apic mode is set (if supported) uint64_t apic_base = read_msr(X86_MSR_IA32_APIC_BASE); apic_base |= (1u<<11); @@ -233,7 +261,15 @@ void lapic_init_percpu(uint level) { } write_msr(X86_MSR_IA32_APIC_BASE, apic_base); - lapic_timer_init_percpu(); + // set the spurious vector register + uint32_t svr = (LAPIC_INT_SPURIOUS | (1u<<8)); // enable + lapic_write(LAPIC_SVR, svr); + + TRACEF("lapic svr %#x\n", lapic_read(LAPIC_SVR)); + + register_int_handler_msi(LAPIC_INT_SPURIOUS, &lapic_spurious_handler, NULL, false); + register_int_handler_msi(LAPIC_INT_GENERIC, &lapic_generic_handler, NULL, false); + register_int_handler_msi(LAPIC_INT_RESCHEDULE, &lapic_reschedule_handler, NULL, false); } LK_INIT_HOOK_FLAGS(lapic_init_percpu, lapic_init_percpu, LK_INIT_LEVEL_VM, LK_INIT_FLAG_SECONDARY_CPUS); @@ -246,7 +282,7 @@ static uint32_t lapic_read_current_tick(void) { return lapic_read(LAPIC_TCCR); } -static void lapic_timer_init_percpu(void) { +static void lapic_timer_init_percpu(uint level) { // check for deadline mode if (use_tsc_deadline) { // put the timer in TSC deadline and clear the match register @@ -260,17 +296,16 @@ static void lapic_timer_init_percpu(void) { lapic_write(LAPIC_TICR, 0); } - // register the local apic interrupts + // register the timer interrupt vector register_int_handler_msi(LAPIC_INT_TIMER, &lapic_timer_handler, NULL, false); } +LK_INIT_HOOK_FLAGS(lapic_timer_init_percpu, lapic_timer_init_percpu, LK_INIT_LEVEL_VM + 1, LK_INIT_FLAG_SECONDARY_CPUS); status_t lapic_timer_init(bool invariant_tsc_supported) { if (!lapic_present) { return ERR_NOT_FOUND; } - lapic_cancel_timer(); - // check for deadline mode bool tsc_deadline = x86_feature_test(X86_FEATURE_TSC_DEADLINE); if (invariant_tsc_supported && tsc_deadline) { @@ -292,10 +327,7 @@ status_t lapic_timer_init(bool invariant_tsc_supported) { timebase_to_lapic.l0, timebase_to_lapic.l32); } - lapic_timer_init_percpu(); - - // register the local apic interrupts - register_int_handler_msi(LAPIC_INT_TIMER, &lapic_timer_handler, NULL, false); + lapic_timer_init_percpu(0); return NO_ERROR; } @@ -325,10 +357,25 @@ void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector) { lapic_write_icr((6u << 8) | (startup_vector >> 12), apic_id); } -void lapic_send_ipi(uint32_t apic_id, uint32_t vector) { +void lapic_send_ipi(uint32_t apic_id, mp_ipi_t ipi) { if (!lapic_present) { return; } - lapic_write_icr(vector, apic_id); + LTRACEF("cpu %u target apic_id %#x, ipi %u\n", arch_curr_cpu_num(), apic_id, ipi); + + uint32_t vector; + switch (ipi) { + case MP_IPI_GENERIC: + vector = LAPIC_INT_GENERIC; + break; + case MP_IPI_RESCHEDULE: + vector = LAPIC_INT_RESCHEDULE; + break; + default: + panic("X86: unknown IPI %u\n", ipi); + } + + // send fixed mode, level asserted, no destination shorthand interrupt + lapic_write_icr(vector | (1U << 14), apic_id); } \ No newline at end of file diff --git a/arch/x86/mp.c b/arch/x86/mp.c index 284d3614..36c11e88 100644 --- a/arch/x86/mp.c +++ b/arch/x86/mp.c @@ -18,8 +18,9 @@ #include #include #include +#include -#define LOCAL_TRACE 1 +#define LOCAL_TRACE 0 #if WITH_SMP @@ -58,30 +59,56 @@ void x86_configure_percpu_early(uint cpu_num, uint apic_id) { } status_t arch_mp_send_ipi(mp_cpu_mask_t target, mp_ipi_t ipi) { - LTRACEF("caller %#x target 0x%x, ipi 0x%x\n", arch_curr_cpu_num(), target, ipi); + LTRACEF("cpu %u target 0x%x, ipi 0x%x\n", arch_curr_cpu_num(), target, ipi); - // XXX call into local apic code to send IPI + DEBUG_ASSERT(arch_ints_disabled()); + uint curr_cpu_num = arch_curr_cpu_num(); - PANIC_UNIMPLEMENTED; + // translate the target bitmap to apic id + while (target) { + uint cpu_num = __builtin_ctz(target); + target &= ~(1u << cpu_num); + + // skip the current cpu + if (cpu_num == curr_cpu_num) { + continue; + } + + x86_percpu_t *percpu = x86_get_percpu_for_cpu(cpu_num); + uint32_t apic_id = percpu->apic_id; + + // send the ipi to the target cpu + lapic_send_ipi(apic_id, ipi); + } + + return NO_ERROR; } void arch_mp_init_percpu(void) { } -static uintptr_t x86_get_apic_id_from_hardware(void) { - // read the apic id out of the hardware - return read_msr(X86_MSR_IA32_APIC_BASE) >> 24; +uint32_t x86_get_apic_id_from_hardware(void) { + // read the apic id out of cpuid leaf 1, which should be present if SMP is enabled. + uint32_t apic_id, unused; + cpuid(0x1, &unused, &apic_id, &unused, &unused); + + apic_id >>= 24; + + // TODO: read full 32bit apic id from x2apic msr if available + + return apic_id; } void x86_secondary_entry(uint cpu_num) { - x86_configure_percpu_early(cpu_num, x86_get_apic_id_from_hardware()); + uint32_t apic_id = x86_get_apic_id_from_hardware(); + x86_configure_percpu_early(cpu_num, apic_id); x86_early_init_percpu(); // run early secondary cpu init routines up to the threading level lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); - dprintf(INFO, "SMP: secondary cpu %u started\n", arch_curr_cpu_num()); + dprintf(INFO, "SMP: secondary cpu %u started, apic id %u\n", arch_curr_cpu_num(), apic_id); lk_secondary_cpu_entry(); diff --git a/arch/x86/pv.c b/arch/x86/pv.c new file mode 100644 index 00000000..fdf8b40f --- /dev/null +++ b/arch/x86/pv.c @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2025 Travis Geiselbrecht + * + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT + */ diff --git a/arch/x86/rules.mk b/arch/x86/rules.mk index c6d83503..a6b31948 100644 --- a/arch/x86/rules.mk +++ b/arch/x86/rules.mk @@ -3,6 +3,7 @@ LOCAL_DIR := $(GET_LOCAL_DIR) MODULE := $(LOCAL_DIR) MODULE_OPTIONS := extra_warnings +MODULE_DEPS := lib/fixed_point # x86 code always runs with the mmu enabled WITH_KERNEL_VM := 1 @@ -74,7 +75,9 @@ MODULE_SRCS += \ $(LOCAL_DIR)/descriptor.c \ $(LOCAL_DIR)/faults.c \ $(LOCAL_DIR)/feature.c \ + $(LOCAL_DIR)/lapic.c \ $(LOCAL_DIR)/mp.c \ + $(LOCAL_DIR)/pv.c \ $(LOCAL_DIR)/thread.c \ # legacy x86's dont have fpu support diff --git a/platform/pc/include/platform/pc/timer.h b/platform/pc/include/platform/pc/timer.h new file mode 100644 index 00000000..a06aceff --- /dev/null +++ b/platform/pc/include/platform/pc/timer.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2025 Travis Geiselbrecht + * + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT + */ +#pragma once + +#include +#include + +// A few shared timer routines needed by the arch/x86 layer +uint32_t pit_calibrate_lapic(uint32_t (*lapic_read_tick)(void)); +uint64_t time_to_tsc_ticks(lk_time_t time); diff --git a/platform/pc/interrupts.c b/platform/pc/interrupts.c index 89dad926..47c6fbaf 100644 --- a/platform/pc/interrupts.c +++ b/platform/pc/interrupts.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "platform_p.h" #include diff --git a/platform/pc/mp.c b/platform/pc/mp.c index 2fc4908c..85ffdd7f 100644 --- a/platform/pc/mp.c +++ b/platform/pc/mp.c @@ -14,6 +14,7 @@ #include #include #include +#include #if WITH_SMP diff --git a/platform/pc/pit.c b/platform/pc/pit.c index 0b57f5f2..f0f7c97c 100644 --- a/platform/pc/pit.c +++ b/platform/pc/pit.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "platform_p.h" #include #include diff --git a/platform/pc/platform_p.h b/platform/pc/platform_p.h index 7e945aab..bd6e428a 100644 --- a/platform/pc/platform_p.h +++ b/platform/pc/platform_p.h @@ -22,19 +22,6 @@ void pic_enable(unsigned int vector, bool enable); void pic_eoi(unsigned int vector); void pic_mask_interrupts(void); -// local apic -void lapic_init(void); -status_t lapic_timer_init(bool invariant_tsc_supported); -void lapic_eoi(unsigned int vector); -void lapic_send_init_ipi(uint32_t apic_id, bool level); -void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector); -void lapic_send_ipi(uint32_t apic_id, uint32_t vector); - -status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval); -void lapic_cancel_timer(void); - -uint64_t time_to_tsc_ticks(lk_time_t time); - // programable interval timer void pit_init(void); status_t pit_set_periodic_timer(platform_timer_callback callback, void *arg, lk_time_t interval); @@ -44,7 +31,6 @@ void pit_stop_timer(void); lk_time_t pit_current_time(void); lk_bigtime_t pit_current_time_hires(void); uint64_t pit_calibrate_tsc(void); -uint32_t pit_calibrate_lapic(uint32_t (*lapic_read_tick)(void)); // secondary cpus void platform_start_secondary_cpus(void); diff --git a/platform/pc/rules.mk b/platform/pc/rules.mk index dbee0748..2ddfd39c 100644 --- a/platform/pc/rules.mk +++ b/platform/pc/rules.mk @@ -22,7 +22,6 @@ MODULE_SRCS += \ $(LOCAL_DIR)/ide.c \ $(LOCAL_DIR)/interrupts.c \ $(LOCAL_DIR)/keyboard.c \ - $(LOCAL_DIR)/lapic.c \ $(LOCAL_DIR)/mp.c \ $(LOCAL_DIR)/mp-boot.S \ $(LOCAL_DIR)/pic.c \ diff --git a/platform/pc/timer.c b/platform/pc/timer.c index 71460e22..ec50e712 100644 --- a/platform/pc/timer.c +++ b/platform/pc/timer.c @@ -16,12 +16,15 @@ #include #include #include -#include "platform_p.h" +#include #include #include +#include #include #include +#include "platform_p.h" + #define LOCAL_TRACE 0 // Deals with all of the various clock sources and event timers on the PC platform.