From d1a332891c17a07305d9b45d3f6a1344dce31938 Mon Sep 17 00:00:00 2001 From: Travis Geiselbrecht Date: Tue, 17 Dec 2024 23:57:56 -0800 Subject: [PATCH] [arch][x86] add x2apic mode to the local apic driver Fill in some more x86 feature bits while at it. --- arch/x86/include/arch/x86.h | 1 + arch/x86/include/arch/x86/feature.h | 104 +++++++++++++++++++++++++++- platform/pc/lapic.c | 86 ++++++++++++++++------- 3 files changed, 165 insertions(+), 26 deletions(-) diff --git a/arch/x86/include/arch/x86.h b/arch/x86/include/arch/x86.h index 1930d2a6..585c2b2f 100644 --- a/arch/x86/include/arch/x86.h +++ b/arch/x86/include/arch/x86.h @@ -199,6 +199,7 @@ typedef tss_64_t tss_t; #define X86_MSR_IA32_PM_ENABLE 0x00000770 /* enable/disable HWP */ #define X86_MSR_IA32_HWP_CAPABILITIES 0x00000771 /* HWP performance range enumeration */ #define X86_MSR_IA32_HWP_REQUEST 0x00000774 /* power manage control hints */ +#define X86_MSR_IA32_X2APIC_BASE 0x00000800 /* X2APIC base register */ #define X86_MSR_IA32_EFER 0xc0000080 /* EFER */ #define X86_MSR_IA32_STAR 0xc0000081 /* system call address */ #define X86_MSR_IA32_LSTAR 0xc0000082 /* long mode call address */ diff --git a/arch/x86/include/arch/x86/feature.h b/arch/x86/include/arch/x86/feature.h index afc12bac..f32d7bb8 100644 --- a/arch/x86/include/arch/x86/feature.h +++ b/arch/x86/include/arch/x86/feature.h @@ -182,71 +182,171 @@ static inline bool x86_feature_test(struct x86_cpuid_bit bit) { /* add feature bits to test here */ /* format: X86_CPUID_BIT(cpuid leaf, register (eax-edx:0-3), bit) */ #define X86_FEATURE_SSE3 X86_CPUID_BIT(0x1, 2, 0) +#define X86_FEATURE_PCLMULQDQ X86_CPUID_BIT(0x1, 2, 1) +#define X86_FEATURE_DTES64 X86_CPUID_BIT(0x1, 2, 2) #define X86_FEATURE_MON X86_CPUID_BIT(0x1, 2, 3) +#define X86_FEATURE_DSCPL X86_CPUID_BIT(0x1, 2, 4) #define X86_FEATURE_VMX X86_CPUID_BIT(0x1, 2, 5) +#define X86_FEATURE_SMX X86_CPUID_BIT(0x1, 2, 6) +#define X86_FEATURE_EIST X86_CPUID_BIT(0x1, 2, 7) #define X86_FEATURE_TM2 X86_CPUID_BIT(0x1, 2, 8) #define X86_FEATURE_SSSE3 X86_CPUID_BIT(0x1, 2, 9) +#define X86_FEATURE_CNXT_ID X86_CPUID_BIT(0x1, 2, 10) +#define X86_FEATURE_SDBG X86_CPUID_BIT(0x1, 2, 11) +#define X86_FEATURE_FMA X86_CPUID_BIT(0x1, 2, 12) +#define X86_FEATURE_CMPXCHG16B X86_CPUID_BIT(0x1, 2, 13) +#define X86_FEATURE_XTPR X86_CPUID_BIT(0x1, 2, 14) #define X86_FEATURE_PDCM X86_CPUID_BIT(0x1, 2, 15) #define X86_FEATURE_PCID X86_CPUID_BIT(0x1, 2, 17) +#define X86_FEATURE_DCA X86_CPUID_BIT(0x1, 2, 18) #define X86_FEATURE_SSE4_1 X86_CPUID_BIT(0x1, 2, 19) #define X86_FEATURE_SSE4_2 X86_CPUID_BIT(0x1, 2, 20) #define X86_FEATURE_X2APIC X86_CPUID_BIT(0x1, 2, 21) +#define X86_FEATURE_MOVBE X86_CPUID_BIT(0x1, 2, 22) +#define X86_FEATURE_POPCNT X86_CPUID_BIT(0x1, 2, 23) #define X86_FEATURE_TSC_DEADLINE X86_CPUID_BIT(0x1, 2, 24) #define X86_FEATURE_AESNI X86_CPUID_BIT(0x1, 2, 25) #define X86_FEATURE_XSAVE X86_CPUID_BIT(0x1, 2, 26) +#define X86_FEATURE_OSXSAVE X86_CPUID_BIT(0x1, 2, 27) #define X86_FEATURE_AVX X86_CPUID_BIT(0x1, 2, 28) #define X86_FEATURE_RDRAND X86_CPUID_BIT(0x1, 2, 30) #define X86_FEATURE_HYPERVISOR X86_CPUID_BIT(0x1, 2, 31) #define X86_FEATURE_FPU X86_CPUID_BIT(0x1, 3, 0) +#define X86_FEATURE_VM86 X86_CPUID_BIT(0x1, 3, 1) +#define X86_FEATURE_DE X86_CPUID_BIT(0x1, 3, 2) #define X86_FEATURE_PSE X86_CPUID_BIT(0x1, 3, 3) +#define X86_FEATURE_TSC X86_CPUID_BIT(0x1, 3, 4) +#define X86_FEATURE_MSR X86_CPUID_BIT(0x1, 3, 5) #define X86_FEATURE_PAE X86_CPUID_BIT(0x1, 3, 6) +#define X86_FEATURE_MCE X86_CPUID_BIT(0x1, 3, 7) +#define X86_FEATURE_CX8 X86_CPUID_BIT(0x1, 3, 8) #define X86_FEATURE_APIC X86_CPUID_BIT(0x1, 3, 9) #define X86_FEATURE_SEP X86_CPUID_BIT(0x1, 3, 11) +#define X86_FEATURE_MTRR X86_CPUID_BIT(0x1, 3, 12) #define X86_FEATURE_PGE X86_CPUID_BIT(0x1, 3, 13) +#define X86_FEATURE_MCA X86_CPUID_BIT(0x1, 3, 14) +#define X86_FEATURE_CMOV X86_CPUID_BIT(0x1, 3, 15) #define X86_FEATURE_PAT X86_CPUID_BIT(0x1, 3, 16) #define X86_FEATURE_PSE36 X86_CPUID_BIT(0x1, 3, 17) +#define X86_FEATURE_PSN X86_CPUID_BIT(0x1, 3, 18) #define X86_FEATURE_CLFLUSH X86_CPUID_BIT(0x1, 3, 19) +#define X86_FEATURE_DS X86_CPUID_BIT(0x1, 3, 21) #define X86_FEATURE_ACPI X86_CPUID_BIT(0x1, 3, 22) #define X86_FEATURE_MMX X86_CPUID_BIT(0x1, 3, 23) #define X86_FEATURE_FXSR X86_CPUID_BIT(0x1, 3, 24) #define X86_FEATURE_SSE X86_CPUID_BIT(0x1, 3, 25) #define X86_FEATURE_SSE2 X86_CPUID_BIT(0x1, 3, 26) +#define X86_FEATURE_SS X86_CPUID_BIT(0x1, 3, 27) +#define X86_FEATURE_HTT X86_CPUID_BIT(0x1, 3, 28) #define X86_FEATURE_TM X86_CPUID_BIT(0x1, 3, 29) +#define X86_FEATURE_PBE X86_CPUID_BIT(0x1, 3, 31) + #define X86_FEATURE_DTS X86_CPUID_BIT(0x6, 0, 0) #define X86_FEATURE_TURBO X86_CPUID_BIT(0x6, 0, 1) +#define X86_FEATURE_ARAT X86_CPUID_BIT(0x6, 0, 2) #define X86_FEATURE_PLN X86_CPUID_BIT(0x6, 0, 4) +#define X86_FEATURE_ECMD X86_CPUID_BIT(0x6, 0, 5) #define X86_FEATURE_PTM X86_CPUID_BIT(0x6, 0, 6) #define X86_FEATURE_HWP X86_CPUID_BIT(0x6, 0, 7) #define X86_FEATURE_HWP_NOT X86_CPUID_BIT(0x6, 0, 8) #define X86_FEATURE_HWP_ACT X86_CPUID_BIT(0x6, 0, 9) #define X86_FEATURE_HWP_PREF X86_CPUID_BIT(0x6, 0, 10) +#define X86_FEATURE_HWP_EPP X86_CPUID_BIT(0x6, 0, 11) +#define X86_FEATURE_HWP_PKG X86_CPUID_BIT(0x6, 0, 12) +#define X86_FEATURE_HDC X86_CPUID_BIT(0x6, 0, 13) #define X86_FEATURE_TURBO_MAX X86_CPUID_BIT(0x6, 0, 14) +#define X86_FEATURE_HWP_CAP X86_CPUID_BIT(0x6, 0, 15) +#define X86_FEATURE_HWP_PECI X86_CPUID_BIT(0x6, 0, 16) +#define X86_FEATURE_HWP_FLEX X86_CPUID_BIT(0x6, 0, 17) +#define X86_FEATURE_HWP_FAST X86_CPUID_BIT(0x6, 0, 18) #define X86_FEATURE_HW_FEEDBACK X86_CPUID_BIT(0x6, 2, 0) #define X86_FEATURE_PERF_BIAS X86_CPUID_BIT(0x6, 2, 3) + #define X86_FEATURE_FSGSBASE X86_CPUID_BIT(0x7, 1, 0) #define X86_FEATURE_TSC_ADJUST X86_CPUID_BIT(0x7, 1, 1) +#define X86_FEATURE_SGX X86_CPUID_BIT(0x7, 1, 2) +#define X86_FEATURE_BMI1 X86_CPUID_BIT(0x7, 1, 3) +#define X86_FEATURE_HLE X86_CPUID_BIT(0x7, 1, 4) #define X86_FEATURE_AVX2 X86_CPUID_BIT(0x7, 1, 5) #define X86_FEATURE_SMEP X86_CPUID_BIT(0x7, 1, 7) +#define X86_FEATURE_BMI2 X86_CPUID_BIT(0x7, 1, 8) #define X86_FEATURE_ERMS X86_CPUID_BIT(0x7, 1, 9) #define X86_FEATURE_INVPCID X86_CPUID_BIT(0x7, 1, 10) +#define X86_FEATURE_RTM X86_CPUID_BIT(0x7, 1, 11) +#define X86_FEATURE_MPX X86_CPUID_BIT(0x7, 1, 14) +#define X86_FEATURE_AVX512F X86_CPUID_BIT(0x7, 1, 16) +#define X86_FEATURE_AVX512DQ X86_CPUID_BIT(0x7, 1, 17) #define X86_FEATURE_RDSEED X86_CPUID_BIT(0x7, 1, 18) +#define X86_FEATURE_ADX X86_CPUID_BIT(0x7, 1, 19) #define X86_FEATURE_SMAP X86_CPUID_BIT(0x7, 1, 20) +#define X86_FEATURE_AVX512IFMA X86_CPUID_BIT(0x7, 1, 21) #define X86_FEATURE_CLFLUSHOPT X86_CPUID_BIT(0x7, 1, 23) #define X86_FEATURE_CLWB X86_CPUID_BIT(0x7, 1, 24) #define X86_FEATURE_PT X86_CPUID_BIT(0x7, 1, 25) +#define X86_FEATURE_AVX512PF X86_CPUID_BIT(0x7, 1, 26) +#define X86_FEATURE_AVX512ER X86_CPUID_BIT(0x7, 1, 27) +#define X86_FEATURE_AVX512CD X86_CPUID_BIT(0x7, 1, 28) +#define X86_FEATURE_SHA X86_CPUID_BIT(0x7, 1, 29) +#define X86_FEATURE_AVX512BW X86_CPUID_BIT(0x7, 1, 30) +#define X86_FEATURE_AVX512VL X86_CPUID_BIT(0x7, 1, 31) +#define X86_FEATURE_PREFETCHWT1 X86_CPUID_BIT(0x7, 2, 0) +#define X86_FEATURE_AVX512VBMI X86_CPUID_BIT(0x7, 2, 1) #define X86_FEATURE_UMIP X86_CPUID_BIT(0x7, 2, 2) #define X86_FEATURE_PKU X86_CPUID_BIT(0x7, 2, 3) +#define X86_FEATURE_OSPKE X86_CPUID_BIT(0x7, 2, 4) +#define X86_FEATURE_WAITPKG X86_CPUID_BIT(0x7, 2, 5) +#define X86_FEATURE_AVX512_VBMI2 X86_CPUID_BIT(0x7, 2, 6) +#define X86_FEATURE_CET_SS X86_CPUID_BIT(0x7, 2, 7) +#define X86_FEATURE_GFNI X86_CPUID_BIT(0x7, 2, 8) +#define X86_FEATURE_VAES X86_CPUID_BIT(0x7, 2, 9) +#define X86_FEATURE_VPCLMULQDQ X86_CPUID_BIT(0x7, 2, 10) +#define X86_FEATURE_AVX512_VNNI X86_CPUID_BIT(0x7, 2, 11) +#define X86_FEATURE_AVX512_BITALG X86_CPUID_BIT(0x7, 2, 12) +#define X86_FEATURE_TIME_EN X86_CPUID_BIT(0x7, 2, 13) +#define X86_FEATURE_AVX512_VPOPCNTDQ X86_CPUID_BIT(0x7, 2, 14) +#define X86_FEATURE_LA57 X86_CPUID_BIT(0x7, 2, 16) +#define X86_FEATURE_RDPID X86_CPUID_BIT(0x7, 2, 22) +#define X86_FEATURE_KL X86_CPUID_BIT(0x7, 2, 23) +#define X86_FEATURE_CLDEMOTE X86_CPUID_BIT(0x7, 2, 25) +#define X86_FEATURE_MOVDIRI X86_CPUID_BIT(0x7, 2, 27) +#define X86_FEATURE_MOVDIR64B X86_CPUID_BIT(0x7, 2, 28) +#define X86_FEATURE_SGX_LC X86_CPUID_BIT(0x7, 2, 30) +#define X86_FEATURE_PKS X86_CPUID_BIT(0x7, 2, 31) +#define X86_FEATURE_AVX512_4VNNIW X86_CPUID_BIT(0x7, 3, 2) +#define X86_FEATURE_AVX512_4FMAPS X86_CPUID_BIT(0x7, 3, 3) +#define X86_FEATURE_FSRM X86_CPUID_BIT(0x7, 3, 4) +#define X86_FEATURE_AVX512_VP2INTERSECT X86_CPUID_BIT(0x7, 3, 8) #define X86_FEATURE_MD_CLEAR X86_CPUID_BIT(0x7, 3, 10) +#define X86_FEATURE_SERIALIZE X86_CPUID_BIT(0x7, 3, 14) +#define X86_FEATURE_HYBRID X86_CPUID_BIT(0x7, 3, 15) +#define X86_FEATURE_PCONFIG X86_CPUID_BIT(0x7, 3, 18) +#define X86_FEATURE_CET_IBT X86_CPUID_BIT(0x7, 3, 20) #define X86_FEATURE_IBRS_IBPB X86_CPUID_BIT(0x7, 3, 26) #define X86_FEATURE_STIBP X86_CPUID_BIT(0x7, 3, 27) #define X86_FEATURE_L1D_FLUSH X86_CPUID_BIT(0x7, 3, 28) #define X86_FEATURE_ARCH_CAPABILITIES X86_CPUID_BIT(0x7, 3, 29) +#define X86_FEATURE_CORE_CAPABILITIES X86_CPUID_BIT(0x7, 3, 30) #define X86_FEATURE_SSBD X86_CPUID_BIT(0x7, 3, 31) -#define X86_FEATURE_KVM_PV_CLOCK X86_CPUID_BIT(0x40000001, 0, 3) +#define X86_FEATURE_KVM_CLOCKSOURCE X86_CPUID_BIT(0x40000000, 0, 0) +#define X86_FEATURE_KVM_NOP_IO_DELAY X86_CPUID_BIT(0x40000000, 0, 1) +#define X86_FEATURE_KVM_MMU_OP X86_CPUID_BIT(0x40000000, 0, 2) +#define X86_FEATURE_KVM_CLOCKSOURCE2 X86_CPUID_BIT(0x40000000, 0, 3) +#define X86_FEATURE_KVM_ASYNC_PF X86_CPUID_BIT(0x40000000, 0, 4) +#define X86_FEATURE_KVM_STEAL_TIME X86_CPUID_BIT(0x40000000, 0, 5) #define X86_FEATURE_KVM_PV_EOI X86_CPUID_BIT(0x40000001, 0, 6) +#define X86_FEATURE_KVM_PV_UNHALT X86_CPUID_BIT(0x40000001, 0, 7) +#define X86_FEATURE_KVM_PV_TLB_FLUSH X86_CPUID_BIT(0x40000001, 0, 9) +#define X86_FEATURE_KVM_ASYNC_PF_VMEXIT X86_CPUID_BIT(0x40000001, 0, 10) #define X86_FEATURE_KVM_PV_IPI X86_CPUID_BIT(0x40000001, 0, 11) -#define X86_FEATURE_KVM_PV_CLOCK_STABLE X86_CPUID_BIT(0x40000001, 0, 24) +#define X86_FEATURE_KVM_POLL_CONTROL X86_CPUID_BIT(0x40000001, 0, 12) +#define X86_FEATURE_KVM_PV_SCHED_YIELD X86_CPUID_BIT(0x40000001, 0, 13) +#define X86_FEATURE_KVM_ASYNC_PF_INT X86_CPUID_BIT(0x40000001, 0, 14) +#define X86_FEATURE_KVM_MSI_EXT_DEST_ID X86_CPUID_BIT(0x40000001, 0, 15) +#define X86_FEATURE_KVM_HC_MAP_GPA_RANGE X86_CPUID_BIT(0x40000001, 0, 16) +#define X86_FEATURE_KVM_MIGRATION_CONTROL X86_CPUID_BIT(0x40000001, 0, 17) +#define X86_FEATURE_KVM_CLOCKSOURCE_STABLE X86_CPUID_BIT(0x40000001, 0, 24) + #define X86_FEATURE_AMD_TOPO X86_CPUID_BIT(0x80000001, 2, 22) #define X86_FEATURE_SSE4A X86_CPUID_BIT(0x80000001, 3, 6) diff --git a/platform/pc/lapic.c b/platform/pc/lapic.c index 45ab1845..decc3adf 100644 --- a/platform/pc/lapic.c +++ b/platform/pc/lapic.c @@ -25,6 +25,7 @@ #define LOCAL_TRACE 1 static bool lapic_present = false; +static bool lapic_x2apic = false; static volatile uint32_t *lapic_mmio; // local apic registers @@ -69,13 +70,38 @@ enum lapic_regs { }; static uint32_t lapic_read(enum lapic_regs reg) { - return mmio_read32(lapic_mmio + reg / 4); + LTRACEF("reg %#x\n", reg); + DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI); + if (lapic_x2apic) { + // TODO: do we need barriers here? + return read_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10); + } else { + return mmio_read32(lapic_mmio + reg / 4); + } } static void lapic_write(enum lapic_regs reg, uint32_t val) { - mmio_write32(lapic_mmio + reg / 4, val); + LTRACEF("reg %#x val %#x\n", reg, val); + DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI); + if (lapic_x2apic) { + write_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10, val); + } else { + mmio_write32(lapic_mmio + reg / 4, val); + } } +// special case to write to the ICR register +static void lapic_write_icr(uint32_t low, uint32_t apic_id) { + LTRACEF("%#x apic_id %#x\n", low, apic_id); + if (lapic_x2apic) { + write_msr(X86_MSR_IA32_X2APIC_BASE + 0x30, ((uint64_t)apic_id << 32) | low); + } else { + lapic_write(LAPIC_ICRHI, apic_id << 24); + lapic_write(LAPIC_ICRLO, low); + } +} + + void lapic_init(void) { // discover the presence of the local apic and map it LTRACE_ENTRY; @@ -91,23 +117,33 @@ void lapic_init_postvm(uint level) { dprintf(INFO, "X86: local apic detected\n"); // IA32_APIC_BASE_MSR - uint64_t apic_base = read_msr(0x1b); + uint64_t apic_base = read_msr(X86_MSR_IA32_APIC_BASE); LTRACEF("raw apic base msr %#llx\n", apic_base); // make sure it's enabled - if ((apic_base & 0x800) == 0) { + if ((apic_base & (1u<<11)) == 0) { dprintf(INFO, "X86: enabling lapic\n"); - apic_base |= 0x800; + apic_base |= (1u<<11); write_msr(0x1b, apic_base); } - apic_base &= ~0xfff; - dprintf(INFO, "X86: lapic physical address %#llx\n", apic_base); + dprintf(INFO, "X86: lapic physical address %#llx\n", apic_base & ~0xfff); + + // see if x2APIC mode is supported and enable + if (x86_feature_test(X86_FEATURE_X2APIC)) { + lapic_x2apic = true; + dprintf(INFO, "X86: local apic supports x2APIC mode\n"); + + write_msr(X86_MSR_IA32_APIC_BASE, apic_base | (1u<<10)); + } // map the lapic into the kernel since it's not guaranteed that the physmap covers it - status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "lapic", PAGE_SIZE, (void **)&lapic_mmio, 0, - apic_base & ~0xfff, /* vmm_flags */ 0, ARCH_MMU_FLAG_UNCACHED_DEVICE); - ASSERT(err == NO_ERROR); + if (!lapic_mmio) { + dprintf(INFO, "X86: mapping lapic into kernel\n"); + status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "lapic", PAGE_SIZE, (void **)&lapic_mmio, 0, + apic_base & ~0xfff, /* vmm_flags */ 0, ARCH_MMU_FLAG_UNCACHED_DEVICE); + ASSERT(err == NO_ERROR); + } // Read the local apic id and version and features uint32_t id = lapic_read(LAPIC_ID); @@ -120,37 +156,39 @@ void lapic_init_postvm(uint level) { if (eas) { dprintf(INFO, "X86: local apic EAS features %#x\n", lapic_read(LAPIC_EXT_FEATURES)); } - - } LK_INIT_HOOK(lapic, lapic_init_postvm, LK_INIT_LEVEL_VM); void lapic_eoi(unsigned int vector) { LTRACEF("vector %#x\n", vector); - if (lapic_present) { - lapic_write(LAPIC_EOI, 0); + if (!lapic_present) { + return; } + + lapic_write(LAPIC_EOI, 0); } void lapic_send_init_ipi(uint32_t apic_id, bool level) { - if (lapic_present) { - lapic_write(LAPIC_ICRHI, apic_id << 24); - lapic_write(LAPIC_ICRLO, (5u << 8) | (level ? (1u << 14) : 0)); + if (!lapic_present) { + return; } + + lapic_write_icr((5u << 8) | (level ? (1u << 14) : 0), apic_id); } void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector) { - if (lapic_present) { - lapic_write(LAPIC_ICRHI, apic_id << 24); - lapic_write(LAPIC_ICRLO, (6u << 8) | (startup_vector >> 12)); + if (!lapic_present) { + return; } + + lapic_write_icr((6u << 8) | (startup_vector >> 12), apic_id); } void lapic_send_ipi(uint32_t apic_id, uint32_t vector) { - if (lapic_present) { - lapic_write(LAPIC_ICRHI, apic_id << 24); - // XXX add correct flag bits - lapic_write(LAPIC_ICRLO, vector); + if (!lapic_present) { + return; } + + lapic_write_icr(vector, apic_id); } \ No newline at end of file