[arch][x86] start of an ioapic driver

Doesn't do much but provided the detection path for it and ability to
hold initialized state. The higher level platform code is going to need
to use it directly so will mostly just provide an api for access to it.

Moved ACPI sniffing back to just after the VM is initialized instead of
all the way into platform_init(). This should try to ensure that all
drivers that come up afterwards will have ioapics discovered in case
future development tries to enable and use them, kicking the machine out
of virtual-wire-mode.

arch/x86/include/arch/x86/apic.h

@@ -15,6 +15,7 @@
 
 // local apic
 void lapic_init(void);
+void lapic_init_postvm(void);
 status_t lapic_timer_init(bool invariant_tsc_supported);
 void lapic_eoi(unsigned int vector);
 void lapic_send_init_ipi(uint32_t apic_id, bool level);
@@ -30,3 +31,5 @@ uint32_t lapic_get_apic_id(void);
 status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval);
 void lapic_cancel_timer(void);
 
+// io apic(s)
+status_t ioapic_init(int index,paddr_t phys_addr, uint apic_id, uint gsi_base);

arch/x86/ioapic.c

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2021 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#include "arch/x86/apic.h"
+
+#include <sys/types.h>
+#include <lk/debug.h>
+#include <lk/err.h>
+#include <lk/reg.h>
+#include <lk/trace.h>
+#include <lk/init.h>
+#include <assert.h>
+#include <arch/x86.h>
+#include <kernel/spinlock.h>
+#include <kernel/vm.h>
+
+#define LOCAL_TRACE 0
+
+enum ioapic_mmio_regs {
+    IOAPIC_REGSEL = 0x00,
+    IOAPIC_IOWIN = 0x10 / 4,
+};
+
+enum ioapic_regs {
+    IOAPIC_ID = 0x00,
+    IOAPIC_VERSION = 0x01,
+    IOAPIC_ARB = 0x02,
+    IOAPIC_REDIR_TABLE_BASE = 0x10,
+};
+
+struct ioapic {
+    paddr_t phys_addr;
+    volatile uint32_t *mmio;
+    uint apic_id;
+    uint gsi_base;
+    uint num_redir_entries;
+
+    // TODO: spinlock for this ioapic
+};
+
+static struct ioapic *ioapics = NULL;
+static size_t num_ioapics = 0;
+
+static uint32_t ioapic_read(struct ioapic *ioapic, enum ioapic_regs reg) {
+    mmio_write32(ioapic->mmio + IOAPIC_REGSEL, reg);
+    return mmio_read32(ioapic->mmio + IOAPIC_IOWIN);
+}
+
+static uint32_t ioapic_write(struct ioapic *ioapic, enum ioapic_regs reg, uint32_t val) {
+    mmio_write32(ioapic->mmio + IOAPIC_REGSEL, reg);
+    mmio_write32(ioapic->mmio + IOAPIC_IOWIN, val);
+    return 0;
+}
+
+status_t ioapic_init(int index, paddr_t phys_addr, uint apic_id, uint gsi_base) {
+    LTRACEF("%d: phys_addr %#lx apic_id %u gsi_base %u\n", index, phys_addr, apic_id, gsi_base);
+
+    {
+        struct ioapic *new_ioapics = realloc(ioapics, sizeof(struct ioapic) * (num_ioapics + 1));
+        if (!new_ioapics) {
+            return ERR_NO_MEMORY;
+        }
+        ioapics = new_ioapics;
+    }
+    struct ioapic *ioapic = &ioapics[num_ioapics];
+
+    LTRACEF("mapping lapic into kernel\n");
+    status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "ioapic", PAGE_SIZE, (void **)&ioapic->mmio, 0,
+                            phys_addr, /* vmm_flags */ 0, ARCH_MMU_FLAG_UNCACHED_DEVICE);
+    if (err != NO_ERROR) {
+        // TODO: free up the newly extended ioapic struct
+        return err;
+    }
+    DEBUG_ASSERT(ioapic->mmio != NULL);
+
+    ioapic->phys_addr = phys_addr;
+    ioapic->apic_id = apic_id;
+    ioapic->gsi_base = gsi_base;
+
+    // probe the capabilities of the ioapic
+    const uint32_t id = ioapic_read(ioapic, IOAPIC_ID) >> 24;
+    uint32_t version = ioapic_read(ioapic, IOAPIC_VERSION);
+    const uint32_t max_redir = (version >> 16) & 0xff;
+    dprintf(INFO, "X86: ioapic %d id %#x version %#x max redir %u\n", index, id, version & 0xff, max_redir);
+
+    ioapic->num_redir_entries = max_redir + 1;
+
+    if (LOCAL_TRACE) {
+        for (uint i = 0; i < ioapic->num_redir_entries; i++) {
+            // read and dump the current entry
+            uint32_t lo = ioapic_read(ioapic, IOAPIC_REDIR_TABLE_BASE + i * 2);
+            uint32_t hi = ioapic_read(ioapic, IOAPIC_REDIR_TABLE_BASE + i * 2 + 1);
+            dprintf(INFO, "X86:     redir %2u hi %#x lo %#x\n", i, hi, lo);
+        }
+    }
+
+    // add it to the global list of ioapics
+    num_ioapics++;
+
+    return NO_ERROR;
+}

arch/x86/lapic.c

@@ -5,7 +5,7 @@
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT
  */
-#include "arch/x86/lapic.h"
+#include "arch/x86/apic.h"
 
 #include <sys/types.h>
 #include <lk/debug.h>
@@ -215,7 +215,7 @@ void lapic_init(void) {
 }
 
 // run on the boot cpu after vm is initialized
-static void lapic_init_postvm(uint level) {
+void lapic_init_postvm(void) {
     if (!lapic_present) {
         return;
     }
@@ -267,7 +267,6 @@ static void lapic_init_postvm(uint level) {
     // Finish up some local initialization that all cpus will want to do
     lapic_init_percpu(0);
 }
-LK_INIT_HOOK(lapic_init_postvm, lapic_init_postvm, LK_INIT_LEVEL_VM + 1);
 
 static void lapic_init_percpu(uint level) {
     // If we're on a secondary cpu we should have a local apic detected and present

arch/x86/mp.c

@@ -18,7 +18,7 @@
 #include <arch/x86/descriptor.h>
 #include <arch/arch_ops.h>
 #include <sys/types.h>
-#include <arch/x86/lapic.h>
+#include <arch/x86/apic.h>
 #include <arch/x86/feature.h>
 
 #define LOCAL_TRACE 0

arch/x86/rules.mk

@@ -76,6 +76,7 @@ MODULE_SRCS += \
 	$(LOCAL_DIR)/descriptor.c \
 	$(LOCAL_DIR)/faults.c \
 	$(LOCAL_DIR)/feature.c \
+	$(LOCAL_DIR)/ioapic.c \
 	$(LOCAL_DIR)/lapic.c \
 	$(LOCAL_DIR)/mp.c \
 	$(LOCAL_DIR)/pv.c \

platform/pc/interrupts.c

@@ -16,13 +16,20 @@
 #include <platform/interrupts.h>
 #include <arch/ops.h>
 #include <arch/x86.h>
-#include <arch/x86/lapic.h>
+#include <arch/x86/apic.h>
 #include <kernel/spinlock.h>
 #include "platform_p.h"
 #include <platform/pc.h>
 
+#if WITH_LIB_ACPI_LITE
+#include <lib/acpi_lite.h>
+#endif
+
 #define LOCAL_TRACE 0
 
+
+// TODO: handle ioapics
+
 static spin_lock_t lock;
 
 #define INTC_TYPE_INTERNAL  0
@@ -209,3 +216,24 @@ status_t platform_compute_msi_values(unsigned int vector, unsigned int cpu, bool
     return NO_ERROR;
 }
 
+// Try to detect the ioapic(s) from ACPI and initialize them
+#if WITH_LIB_ACPI_LITE
+static void io_apic_callback(const void *_entry, size_t entry_len, void *cookie) {
+    const struct acpi_madt_io_apic_entry *entry = _entry;
+
+    static int index = 0;
+    ioapic_init(index++, entry->io_apic_address, entry->io_apic_id, entry->global_system_interrupt_base);
+}
+#endif
+
+void platform_init_interrupts_postvm(void) {
+    // Bring up the local apic on the first cpu
+    // Doesn't need ACPI to detect its presence
+    lapic_init_postvm();
+
+#if WITH_LIB_ACPI_LITE
+    // Now that we've scanned ACPI, try to initialize the ioapic(s)
+    acpi_process_madt_entries_etc(ACPI_MADT_TYPE_IO_APIC, &io_apic_callback, NULL);
+#endif
+}
+

platform/pc/mp.c

@@ -10,15 +10,16 @@
 
 #include <kernel/thread.h>
 #include <kernel/vm.h>
-#include <lib/acpi_lite.h>
 #include <lk/err.h>
 #include <lk/main.h>
 #include <lk/trace.h>
 #include <string.h>
-#include <arch/x86/lapic.h>
+#include <arch/x86/apic.h>
 
 #if WITH_SMP
 
+#include <lib/acpi_lite.h>
+
 #define TRAMPOLINE_ADDRESS 0x4000
 
 #define LOCAL_TRACE 1

platform/pc/platform.c

@@ -12,6 +12,7 @@
 #include <lk/init.h>
 #include <lk/trace.h>
 #include <arch/x86/mmu.h>
+#include <arch/x86/apic.h>
 #include <platform.h>
 #include <platform/pc.h>
 #include <platform/console.h>
@@ -24,7 +25,6 @@
 #include <assert.h>
 #include <inttypes.h>
 #include <kernel/vm.h>
-#include <lib/acpi_lite.h>
 
 #include "platform_p.h"
 
@@ -34,6 +34,11 @@
 #if WITH_LIB_MINIP
 #include <lib/minip.h>
 #endif
+#if WITH_LIB_ACPI_LITE
+#include <lib/acpi_lite.h>
+
+static bool found_acpi = false;
+#endif
 
 #define LOCAL_TRACE 0
 
@@ -215,21 +220,33 @@ void platform_early_init(void) {
     dprintf(INFO, "PC: total memory detected %" PRIu64 " bytes\n", total_mem);
 }
 
+// Look for the ACPI tables just after the vm is initialized.
+void platform_init_postvm(uint level) {
+#if WITH_LIB_ACPI_LITE
+    // Look for the root ACPI table
+    status_t err = acpi_lite_init(0);
+    if (err != NO_ERROR) {
+        return;
+    }
+    found_acpi = true;
+
+    if (LOCAL_TRACE) {
+        acpi_lite_dump_tables(false);
+    }
+    acpi_lite_dump_madt_table();
+#endif
+
+    platform_init_interrupts_postvm();
+    platform_init_timer();
+}
+
+LK_INIT_HOOK(platform_init_postvm, platform_init_postvm, LK_INIT_LEVEL_VM);
+
 void platform_init(void) {
     platform_init_debug();
 
     platform_init_keyboard(&console_input_buf);
 
-    // Look for the root ACPI table
-    __UNUSED bool found_acpi = false;
-    if (acpi_lite_init(0) == NO_ERROR) {
-        if (LOCAL_TRACE) {
-            acpi_lite_dump_tables(false);
-        }
-        acpi_lite_dump_madt_table();
-        found_acpi = true;
-    }
-
     // Look for secondary cpus
 #if WITH_SMP
     platform_start_secondary_cpus();

platform/pc/platform_p.h

@@ -15,6 +15,8 @@ extern cbuf_t console_input_buf;
 void platform_init_debug_early(void);
 void platform_init_debug(void);
 void platform_init_interrupts(void);
+void platform_init_interrupts_postvm(void);
+void platform_init_timer(void);
 
 // legacy programmable interrupt controller
 void pic_init(void);

platform/pc/rules.mk

@@ -6,13 +6,13 @@ MODULE := $(LOCAL_DIR)
 # legacy implies older hardware, pre pentium, pre pci
 CPU ?= modern
 
-MODULE_DEPS += lib/acpi_lite
 MODULE_DEPS += lib/bio
 MODULE_DEPS += lib/cbuf
 MODULE_DEPS += lib/fixed_point
 
 ifneq ($(CPU),legacy)
 MODULE_DEPS += dev/bus/pci/drivers
+MODULE_DEPS += lib/acpi_lite
 endif
 
 MODULE_SRCS += \

platform/pc/timer.c

@@ -19,7 +19,7 @@
 #include <platform/pc/timer.h>
 #include <arch/x86.h>
 #include <arch/x86/feature.h>
-#include <arch/x86/lapic.h>
+#include <arch/x86/apic.h>
 #include <arch/x86/pv.h>
 #include <inttypes.h>
 #include <lib/fixed_point.h>
@@ -87,7 +87,7 @@ uint64_t time_to_tsc_ticks(lk_time_t time) {
     return u64_mul_u32_fp32_64(time, timebase_to_tsc);
 }
 
-void pc_init_timer(unsigned int level) {
+void platform_init_timer(void) {
     // Initialize the PIT, it's always present in PC hardware
     pit_init();
     clock_source = CLOCK_SOURCE_PIT;
@@ -160,8 +160,6 @@ out:
     dprintf(INFO, "PC: using %s clock source\n", clock_source_name());
 }
 
-LK_INIT_HOOK(pc_timer, pc_init_timer, LK_INIT_LEVEL_VM + 2);
-
 status_t platform_set_periodic_timer(platform_timer_callback callback, void *arg, lk_time_t interval) {
     if (use_lapic_timer) {
         PANIC_UNIMPLEMENTED;