[arch][x86] start getting inter-processor-interrupts working

-Move the local apic driver to arch/x86
-Add routines to send IPIs between cpus

Something is unstable at the moment and the system crashes after a while
with random corruptions when using SMP.
This commit is contained in:
Travis Geiselbrecht
2025-04-01 00:40:50 -07:00
parent 21ce533327
commit 5a520eca3e
16 changed files with 172 additions and 54 deletions

View File

@@ -83,8 +83,6 @@ FUNCTION(setup_idt)
loop .Lloop
lidt _idtr
ret
END_FUNCTION(setup_idt)

View File

@@ -113,8 +113,6 @@ FUNCTION(setup_idt)
loop .Lloop
lidt _idtr
ret
END_FUNCTION(setup_idt)

View File

@@ -80,6 +80,9 @@ void x86_early_init_percpu(void) {
x86_set_gdt_descriptor(selector, &system_tss, sizeof(system_tss), 1, 0, 0, SEG_TYPE_TSS, 0, 0);
x86_ltr(selector);
/* load the kernel's IDT */
asm("lidt _idtr");
x86_mmu_early_init_percpu();
#if X86_WITH_FPU
x86_fpu_early_init_percpu();

View File

@@ -0,0 +1,26 @@
/*
* Copyright (c) 2025 Travis Geiselbrecht
*
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file or at
* https://opensource.org/licenses/MIT
*/
#pragma once
#include <platform/timer.h>
#include <stdbool.h>
#include <sys/types.h>
#include <kernel/mp.h>
// local apic
void lapic_init(void);
status_t lapic_timer_init(bool invariant_tsc_supported);
void lapic_eoi(unsigned int vector);
void lapic_send_init_ipi(uint32_t apic_id, bool level);
void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector);
void lapic_send_ipi(uint32_t apic_id, mp_ipi_t ipi);
status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval);
void lapic_cancel_timer(void);

View File

@@ -16,7 +16,7 @@ typedef struct x86_percpu {
struct x86_percpu *self;
uint cpu_num;
uint apic_id;
uint32_t apic_id;
struct thread *current_thread;
@@ -70,10 +70,13 @@ static inline uint x86_get_cpu_num(void) {
}
// get the current apic id
static inline uint x86_get_apic_id(void) {
static inline uint32_t x86_get_apic_id(void) {
return x86_read_gs_offset32(X86_PERCPU_FIELD_OFFSET(apic_id));
}
// read it from hardware directly
uint32_t x86_get_apic_id_from_hardware(void);
// get/set the current thread
struct thread;
@@ -83,4 +86,4 @@ static inline struct thread *x86_get_current_thread(void) {
static inline void x86_set_current_thread(struct thread *t) {
x86_write_gs_offset_ptr(X86_PERCPU_FIELD_OFFSET(current_thread), t);
}
}

381
arch/x86/lapic.c Normal file
View File

@@ -0,0 +1,381 @@
/*
* Copyright (c) 2021 Travis Geiselbrecht
*
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file or at
* https://opensource.org/licenses/MIT
*/
#include "arch/x86/lapic.h"
#include <sys/types.h>
#include <lk/debug.h>
#include <lk/err.h>
#include <lk/reg.h>
#include <lk/trace.h>
#include <lk/init.h>
#include <lib/fixed_point.h>
#include <assert.h>
#include <kernel/thread.h>
#include <platform/interrupts.h>
#include <arch/ops.h>
#include <arch/x86.h>
#include <arch/x86/feature.h>
#include <kernel/spinlock.h>
#include <platform/time.h>
#include <platform/timer.h>
#include <platform/pc/timer.h>
#include <kernel/vm.h>
#include <kernel/mp.h>
#define LOCAL_TRACE 0
static bool lapic_present = false;
static bool lapic_x2apic = false;
static bool use_tsc_deadline = false;
static volatile uint32_t *lapic_mmio;
static struct fp_32_64 timebase_to_lapic;
// TODO: move these callbacks into the shared timer code
static platform_timer_callback t_callback;
static void *callback_arg;
static void lapic_init_percpu(uint level);
// local apic registers
enum lapic_regs {
LAPIC_ID = 0x20,
LAPIC_VERSION = 0x30,
LAPIC_TPR = 0x80,
LAPIC_APR = 0x90,
LAPIC_PPR = 0xa0,
LAPIC_EOI = 0xb0,
LAPIC_RRD = 0xc0,
LAPIC_LDR = 0xd0,
LAPIC_DFR = 0xe0,
LAPIC_SVR = 0xf0,
LAPIC_ISR0 = 0x100,
LAPIC_TMR0 = 0x180,
LAPIC_IRR0 = 0x200,
LAPIC_ESR = 0x280,
LAPIC_CMCI = 0x2f0,
LAPIC_ICRLO = 0x300,
LAPIC_ICRHI = 0x310,
LAPIC_TIMER = 0x320,
LAPIC_THERMAL = 0x330,
LAPIC_PERF = 0x340,
LAPIC_LINT0 = 0x350,
LAPIC_LINT1 = 0x360,
LAPIC_ERROR = 0x370,
LAPIC_TICR = 0x380,
LAPIC_TCCR = 0x390,
LAPIC_DIV = 0x3e0,
// Extended features
LAPIC_EXT_FEATURES = 0x400,
LAPIC_EXT_CONTROL = 0x410,
LAPIC_EXT_SEOI = 0x420,
LAPIC_EXT_IER0 = 0x480,
LAPIC_EXT_LVT0 = 0x500,
};
enum lapic_interrupts {
LAPIC_INT_TIMER = 0xf8,
LAPIC_INT_SPURIOUS,
LAPIC_INT_GENERIC,
LAPIC_INT_RESCHEDULE,
};
enum lapic_timer_mode {
LAPIC_TIMER_MODE_ONESHOT = 0,
LAPIC_TIMER_MODE_PERIODIC = 1,
LAPIC_TIMER_MODE_TSC_DEADLINE = 2,
};
static uint32_t lapic_read(enum lapic_regs reg) {
LTRACEF_LEVEL(2, "reg %#x\n", reg);
DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI);
if (lapic_x2apic) {
// TODO: do we need barriers here?
return read_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10);
} else {
return mmio_read32(lapic_mmio + reg / 4);
}
}
static void lapic_write(enum lapic_regs reg, uint32_t val) {
LTRACEF_LEVEL(2, "reg %#x val %#x\n", reg, val);
DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI);
if (lapic_x2apic) {
write_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10, val);
} else {
mmio_write32(lapic_mmio + reg / 4, val);
}
}
// special case to write to the ICR register
static void lapic_write_icr(uint32_t low, uint32_t apic_id) {
LTRACEF_LEVEL(2, "%#x apic_id %#x\n", low, apic_id);
if (lapic_x2apic) {
write_msr(X86_MSR_IA32_X2APIC_BASE + 0x30, ((uint64_t)apic_id << 32) | low);
} else {
lapic_write(LAPIC_ICRHI, apic_id << 24);
lapic_write(LAPIC_ICRLO, low);
}
}
status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval) {
LTRACEF("cpu %u interval %u\n", arch_curr_cpu_num(), interval);
DEBUG_ASSERT(arch_ints_disabled());
t_callback = callback;
callback_arg = arg;
if (use_tsc_deadline) {
uint64_t now = __builtin_ia32_rdtsc();
uint64_t delta = time_to_tsc_ticks(interval);
uint64_t deadline = now + delta;
LTRACEF("now %llu delta %llu deadline %llu\n", now, delta, deadline);
write_msr(X86_MSR_IA32_TSC_DEADLINE, deadline);
} else {
// set the initial count, which should trigger the timer
uint64_t ticks = u64_mul_u32_fp32_64(interval, timebase_to_lapic);
if (ticks > UINT32_MAX) {
ticks = UINT32_MAX;
}
lapic_write(LAPIC_TICR, ticks & 0xffffffff);
}
return NO_ERROR;
}
void lapic_cancel_timer(void) {
LTRACE;
DEBUG_ASSERT(arch_ints_disabled());
if (use_tsc_deadline) {
write_msr(X86_MSR_IA32_TSC_DEADLINE, 0);
} else {
lapic_write(LAPIC_TICR, 0);
}
}
static enum handler_return lapic_timer_handler(void *arg) {
LTRACEF("cpu %u\n", arch_curr_cpu_num());
enum handler_return ret = INT_NO_RESCHEDULE;
if (t_callback) {
ret = t_callback(callback_arg, current_time());
}
return ret;
}
static enum handler_return lapic_spurious_handler(void *arg) {
LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg);
return INT_NO_RESCHEDULE;
}
static enum handler_return lapic_generic_handler(void *arg) {
LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg);
return INT_NO_RESCHEDULE;
}
static enum handler_return lapic_reschedule_handler(void *arg) {
LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg);
return mp_mbx_reschedule_irq();
}
void lapic_init(void) {
lapic_present = x86_feature_test(X86_FEATURE_APIC);
}
static void lapic_init_postvm(uint level) {
if (!lapic_present) {
return;
}
dprintf(INFO, "X86: local apic detected\n");
// IA32_APIC_BASE_MSR
uint64_t apic_base = read_msr(X86_MSR_IA32_APIC_BASE);
LTRACEF("raw apic base msr %#llx\n", apic_base);
// make sure it's enabled
if ((apic_base & (1u<<11)) == 0) {
dprintf(INFO, "X86: enabling lapic\n");
apic_base |= (1u<<11);
write_msr(X86_MSR_IA32_APIC_BASE, apic_base);
}
dprintf(INFO, "X86: lapic physical address %#llx\n", apic_base & ~0xfff);
// see if x2APIC mode is supported and enable
if (x86_feature_test(X86_FEATURE_X2APIC)) {
lapic_x2apic = true;
dprintf(INFO, "X86: local apic supports x2APIC mode\n");
write_msr(X86_MSR_IA32_APIC_BASE, apic_base | (1u<<10));
}
// map the lapic into the kernel since it's not guaranteed that the physmap covers it
if (!lapic_mmio) {
LTRACEF("mapping lapic into kernel\n");
status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "lapic", PAGE_SIZE, (void **)&lapic_mmio, 0,
apic_base & ~0xfff, /* vmm_flags */ 0, ARCH_MMU_FLAG_UNCACHED_DEVICE);
ASSERT(err == NO_ERROR);
}
// Read the local apic id and version and features
uint32_t id = lapic_read(LAPIC_ID);
uint32_t version = lapic_read(LAPIC_VERSION);
bool eas = version & (1u<<31);
uint32_t max_lvt = (version >> 16) & 0xff;
version &= 0xff;
dprintf(INFO, "X86: local apic id %#x version %#x\n", id, version);
dprintf(INFO, "X86: local apic max lvt entries %u\n", max_lvt);
if (eas) {
dprintf(INFO, "X86: local apic EAS features %#x\n", lapic_read(LAPIC_EXT_FEATURES));
}
// Finish up some local initialization that all cpus will want to do
lapic_init_percpu(0);
}
LK_INIT_HOOK(lapic_init_postvm, lapic_init_postvm, LK_INIT_LEVEL_VM + 1);
static void lapic_init_percpu(uint level) {
// Make sure the apic is enabled and x2apic mode is set (if supported)
uint64_t apic_base = read_msr(X86_MSR_IA32_APIC_BASE);
apic_base |= (1u<<11);
if (lapic_x2apic) {
apic_base |= (1u<<10);
}
write_msr(X86_MSR_IA32_APIC_BASE, apic_base);
// set the spurious vector register
uint32_t svr = (LAPIC_INT_SPURIOUS | (1u<<8)); // enable
lapic_write(LAPIC_SVR, svr);
TRACEF("lapic svr %#x\n", lapic_read(LAPIC_SVR));
register_int_handler_msi(LAPIC_INT_SPURIOUS, &lapic_spurious_handler, NULL, false);
register_int_handler_msi(LAPIC_INT_GENERIC, &lapic_generic_handler, NULL, false);
register_int_handler_msi(LAPIC_INT_RESCHEDULE, &lapic_reschedule_handler, NULL, false);
}
LK_INIT_HOOK_FLAGS(lapic_init_percpu, lapic_init_percpu, LK_INIT_LEVEL_VM, LK_INIT_FLAG_SECONDARY_CPUS);
static uint32_t lapic_read_current_tick(void) {
if (!lapic_present) {
return 0;
}
return lapic_read(LAPIC_TCCR);
}
static void lapic_timer_init_percpu(uint level) {
// check for deadline mode
if (use_tsc_deadline) {
// put the timer in TSC deadline and clear the match register
uint32_t val = (LAPIC_TIMER_MODE_TSC_DEADLINE << 17) | LAPIC_INT_TIMER;
lapic_write(LAPIC_TIMER, val);
write_msr(X86_MSR_IA32_TSC_DEADLINE, 0);
} else {
// configure the local timer and make sure it is not set to fire
uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER;
lapic_write(LAPIC_TIMER, val);
lapic_write(LAPIC_TICR, 0);
}
// register the timer interrupt vector
register_int_handler_msi(LAPIC_INT_TIMER, &lapic_timer_handler, NULL, false);
}
LK_INIT_HOOK_FLAGS(lapic_timer_init_percpu, lapic_timer_init_percpu, LK_INIT_LEVEL_VM + 1, LK_INIT_FLAG_SECONDARY_CPUS);
status_t lapic_timer_init(bool invariant_tsc_supported) {
if (!lapic_present) {
return ERR_NOT_FOUND;
}
// check for deadline mode
bool tsc_deadline = x86_feature_test(X86_FEATURE_TSC_DEADLINE);
if (invariant_tsc_supported && tsc_deadline) {
dprintf(INFO, "X86: local apic timer supports TSC deadline mode\n");
use_tsc_deadline = true;
} else {
// configure the local timer and make sure it is not set to fire
uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER;
lapic_write(LAPIC_TIMER, val);
// calibrate the timer frequency
lapic_write(LAPIC_TICR, 0xffffffff); // countdown from the max count
uint32_t lapic_hz = pit_calibrate_lapic(&lapic_read_current_tick);
lapic_write(LAPIC_TICR, 0);
printf("X86: local apic timer frequency %uHz\n", lapic_hz);
fp_32_64_div_32_32(&timebase_to_lapic, lapic_hz, 1000);
dprintf(INFO, "X86: timebase to local apic timer ratio %u.%08u...\n",
timebase_to_lapic.l0, timebase_to_lapic.l32);
}
lapic_timer_init_percpu(0);
return NO_ERROR;
}
void lapic_eoi(unsigned int vector) {
LTRACEF("vector %#x\n", vector);
if (!lapic_present) {
return;
}
lapic_write(LAPIC_EOI, 0);
}
void lapic_send_init_ipi(uint32_t apic_id, bool level) {
if (!lapic_present) {
return;
}
lapic_write_icr((5u << 8) | (level ? (1u << 14) : 0), apic_id);
}
void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector) {
if (!lapic_present) {
return;
}
lapic_write_icr((6u << 8) | (startup_vector >> 12), apic_id);
}
void lapic_send_ipi(uint32_t apic_id, mp_ipi_t ipi) {
if (!lapic_present) {
return;
}
LTRACEF("cpu %u target apic_id %#x, ipi %u\n", arch_curr_cpu_num(), apic_id, ipi);
uint32_t vector;
switch (ipi) {
case MP_IPI_GENERIC:
vector = LAPIC_INT_GENERIC;
break;
case MP_IPI_RESCHEDULE:
vector = LAPIC_INT_RESCHEDULE;
break;
default:
panic("X86: unknown IPI %u\n", ipi);
}
// send fixed mode, level asserted, no destination shorthand interrupt
lapic_write_icr(vector | (1U << 14), apic_id);
}

View File

@@ -18,8 +18,9 @@
#include <arch/x86/descriptor.h>
#include <arch/arch_ops.h>
#include <sys/types.h>
#include <arch/x86/lapic.h>
#define LOCAL_TRACE 1
#define LOCAL_TRACE 0
#if WITH_SMP
@@ -58,30 +59,56 @@ void x86_configure_percpu_early(uint cpu_num, uint apic_id) {
}
status_t arch_mp_send_ipi(mp_cpu_mask_t target, mp_ipi_t ipi) {
LTRACEF("caller %#x target 0x%x, ipi 0x%x\n", arch_curr_cpu_num(), target, ipi);
LTRACEF("cpu %u target 0x%x, ipi 0x%x\n", arch_curr_cpu_num(), target, ipi);
// XXX call into local apic code to send IPI
DEBUG_ASSERT(arch_ints_disabled());
uint curr_cpu_num = arch_curr_cpu_num();
PANIC_UNIMPLEMENTED;
// translate the target bitmap to apic id
while (target) {
uint cpu_num = __builtin_ctz(target);
target &= ~(1u << cpu_num);
// skip the current cpu
if (cpu_num == curr_cpu_num) {
continue;
}
x86_percpu_t *percpu = x86_get_percpu_for_cpu(cpu_num);
uint32_t apic_id = percpu->apic_id;
// send the ipi to the target cpu
lapic_send_ipi(apic_id, ipi);
}
return NO_ERROR;
}
void arch_mp_init_percpu(void) {
}
static uintptr_t x86_get_apic_id_from_hardware(void) {
// read the apic id out of the hardware
return read_msr(X86_MSR_IA32_APIC_BASE) >> 24;
uint32_t x86_get_apic_id_from_hardware(void) {
// read the apic id out of cpuid leaf 1, which should be present if SMP is enabled.
uint32_t apic_id, unused;
cpuid(0x1, &unused, &apic_id, &unused, &unused);
apic_id >>= 24;
// TODO: read full 32bit apic id from x2apic msr if available
return apic_id;
}
void x86_secondary_entry(uint cpu_num) {
x86_configure_percpu_early(cpu_num, x86_get_apic_id_from_hardware());
uint32_t apic_id = x86_get_apic_id_from_hardware();
x86_configure_percpu_early(cpu_num, apic_id);
x86_early_init_percpu();
// run early secondary cpu init routines up to the threading level
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
dprintf(INFO, "SMP: secondary cpu %u started\n", arch_curr_cpu_num());
dprintf(INFO, "SMP: secondary cpu %u started, apic id %u\n", arch_curr_cpu_num(), apic_id);
lk_secondary_cpu_entry();

7
arch/x86/pv.c Normal file
View File

@@ -0,0 +1,7 @@
/*
* Copyright (c) 2025 Travis Geiselbrecht
*
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file or at
* https://opensource.org/licenses/MIT
*/

View File

@@ -3,6 +3,7 @@ LOCAL_DIR := $(GET_LOCAL_DIR)
MODULE := $(LOCAL_DIR)
MODULE_OPTIONS := extra_warnings
MODULE_DEPS := lib/fixed_point
# x86 code always runs with the mmu enabled
WITH_KERNEL_VM := 1
@@ -74,7 +75,9 @@ MODULE_SRCS += \
$(LOCAL_DIR)/descriptor.c \
$(LOCAL_DIR)/faults.c \
$(LOCAL_DIR)/feature.c \
$(LOCAL_DIR)/lapic.c \
$(LOCAL_DIR)/mp.c \
$(LOCAL_DIR)/pv.c \
$(LOCAL_DIR)/thread.c \
# legacy x86's dont have fpu support