[platform][qemu-riscv] Fixup qemu-riscv target

-Fix plic driver to handle machine vs supervisor mode
-Add switch to scripts/do-qemuriscv to run in supervisor mode (with OpenSBI)
-Use the FDT to detect the number of cpus and size of memory

platform/qemu-virt-riscv/include/platform/virt.h

@@ -14,7 +14,7 @@
 #define IRQ_VIRTIO_BASE 1
 #define IRQ_UART0       10
 #define IRQ_PCIE_BASE   0x20
-#define NUM_IRQS        0x35
+#define NUM_IRQS        127
 
 #define CLINT_BASE  0x02000000
 #define PLIC_BASE   0x0c000000
@@ -22,3 +22,9 @@
 #define VIRTIO_BASE 0x10001000
 #define DRAM_BASE   0x80000000
 
+#if RISCV_XMODE_OFFSET == RISCV_MACH_OFFSET
+#define PLIC_HART_IDX(hart)    (2 * (hart))
+#elif RISCV_XMODE_OFFSET == RISCV_SUPER_OFFSET
+#define PLIC_HART_IDX(hart)    ((2 * (hart)) + 1)
+#endif
+

platform/qemu-virt-riscv/platform.c

@@ -6,6 +6,7 @@
  * https://opensource.org/licenses/MIT
  */
 #include <lk/reg.h>
+#include <lk/trace.h>
 #include <kernel/thread.h>
 #include <kernel/novm.h>
 #include <platform.h>
@@ -18,16 +19,23 @@
 
 #include "platform_p.h"
 
+#define LOCAL_TRACE 1
+
 extern ulong lk_boot_args[4];
 
+static int cpu_count = 0;
+
 void platform_early_init(void) {
     plic_early_init();
 
+    LTRACEF("starting FDT scan\n");
+
     /* look for a flattened device tree in the second arg passed to us */
+    bool found_mem = false;
     const void *fdt = (void *)lk_boot_args[1];
     int err = fdt_check_header(fdt);
     if (err >= 0) {
-        /* walk the nodes, looking for 'memory@*' */
+        /* walk the nodes */
         int depth = 0;
         int offset = 0;
         for (;;) {
@@ -40,23 +48,42 @@ void platform_early_init(void) {
             if (!name)
                 continue;
 
+            LTRACEF_LEVEL(2, "name '%s', depth %d\n", name, depth);
+
             /* look for the 'memory@*' property */
-            if (strncmp(name, "memory@", 7) == 0) {
+            if (!found_mem && strncmp(name, "memory@", 7) == 0 && depth == 1) {
                 int lenp;
                 const void *prop_ptr = fdt_getprop(fdt, offset, "reg", &lenp);
                 if (prop_ptr && lenp == 0x10) {
                     /* we're looking at a memory descriptor */
-                    uint64_t base = fdt64_to_cpu(*(uint64_t *)prop_ptr);
+                    uint64_t base = fdt64_to_cpu(*(const uint64_t *)prop_ptr);
                     uint64_t len = fdt64_to_cpu(*((const uint64_t *)prop_ptr + 1));
 
                     /* add another novm arena */
                     printf("FDT: found memory arena, base %#llx size %#llx\n", base, len);
                     novm_add_arena("fdt", base, len);
-                    break; // stop searching after the first one
+                    found_mem = true; // stop searching after the first one
+                }
+            }
+
+            /* look for a cpu leaf and count the number of cpus */
+            if (strncmp(name, "cpu@", 4) == 0 && depth == 2) {
+                int lenp;
+                const void *prop_ptr = fdt_getprop(fdt, offset, "reg", &lenp);
+                if (prop_ptr && lenp == 0x4) {
+                    uint32_t id = fdt32_to_cpu(*(const uint32_t *)prop_ptr);
+                    printf("FDT: found cpu id %u\n", id);
+                    cpu_count++;
                 }
             }
         }
     }
+
+    if (cpu_count > 0) {
+        riscv_set_secondary_count(cpu_count - 1);
+    }
+
+    LTRACEF("done scanning FDT\n");
 }
 
 void platform_init(void) {

platform/qemu-virt-riscv/plic.c

@@ -20,13 +20,12 @@
 #define LOCAL_TRACE 0
 
 // Driver for PLIC implementation for qemu riscv virt machine
-
+#define PLIC_PRIORITY(irq) (PLIC_BASE + 4 + 4 * (irq))
-#define PLIC_PRIORITY(x) (PLIC_BASE + 4 * (x))
+#define PLIC_PENDING(irq)  (PLIC_BASE + 0x1000 + (4 * ((irq) / 32)))
-#define PLIC_PENDING(x)  (PLIC_BASE + 0x1000 + 4 * ((x) / 32))
+#define PLIC_ENABLE(irq, hart)      (PLIC_BASE + 0x2000 + (0x80 * PLIC_HART_IDX(hart)) + (4 * ((irq) / 32)))
-#define PLIC_ENABLE(x)   (PLIC_BASE + 0x2000 + 4 * ((x) / 32))
+#define PLIC_THRESHOLD(hart)        (PLIC_BASE + 0x200000 + (0x1000 * PLIC_HART_IDX(hart)))
-#define PLIC_THRESHOLD   (PLIC_BASE + 0x200000)
+#define PLIC_COMPLETE(hart)         (PLIC_BASE + 0x200004 + (0x1000 * PLIC_HART_IDX(hart)))
-#define PLIC_COMPLETE    (PLIC_BASE + 0x200004)
+#define PLIC_CLAIM(hart)            PLIC_COMPLETE(hart)
-#define PLIC_CLAIM       PLIC_COMPLETE
 
 static struct int_handlers {
     int_handler handler;
@@ -35,25 +34,26 @@ static struct int_handlers {
 
 void plic_early_init(void) {
     // mask all irqs and set their priority to 1
+    // TODO: mask on all the other cpus too
     for (int i = 1; i < NUM_IRQS; i++) {
-        *REG32(PLIC_ENABLE(i)) &= ~(1 << (i % 32));
+        *REG32(PLIC_ENABLE(i, riscv_current_hart())) &= ~(1 << (i % 32));
         *REG32(PLIC_PRIORITY(i)) = 1;
     }
 
     // set global priority threshold to 0
-    *REG32(PLIC_THRESHOLD) = 0;
+    *REG32(PLIC_THRESHOLD(riscv_current_hart())) = 0;
 }
 
 void plic_init(void) {
 }
 
 status_t mask_interrupt(unsigned int vector) {
-    *REG32(PLIC_ENABLE(vector)) &= ~(1 << (vector % 32));
+    *REG32(PLIC_ENABLE(vector, riscv_current_hart())) &= ~(1 << (vector % 32));
     return NO_ERROR;
 }
 
 status_t unmask_interrupt(unsigned int vector) {
-    *REG32(PLIC_ENABLE(vector)) |= (1 << (vector % 32));
+    *REG32(PLIC_ENABLE(vector, riscv_current_hart())) |= (1 << (vector % 32));
     return NO_ERROR;
 }
 
@@ -68,7 +68,7 @@ void register_int_handler(unsigned int vector, int_handler handler, void *arg) {
 
 enum handler_return riscv_platform_irq(void) {
     // see what irq triggered it
-    uint32_t vector = *REG32(PLIC_CLAIM);
+    uint32_t vector = *REG32(PLIC_CLAIM(riscv_current_hart()));
     LTRACEF("vector %u\n", vector);
 
     if (unlikely(vector == 0)) {
@@ -85,7 +85,7 @@ enum handler_return riscv_platform_irq(void) {
     }
 
     // ack the interrupt
-    *REG32(PLIC_COMPLETE) = vector;
+    *REG32(PLIC_COMPLETE(riscv_current_hart())) = vector;
 
     KEVLOG_IRQ_EXIT(vector);
 

platform/qemu-virt-riscv/rules.mk

@@ -4,6 +4,9 @@ MODULE := $(LOCAL_DIR)
 
 ARCH := riscv
 SUBARCH ?= 32
+RISCV_MODE ?= machine
+WITH_SMP ?= 1
+SMP_MAX_CPUS ?= 8
 
 MODULE_DEPS += lib/cbuf
 MODULE_DEPS += lib/fdt
@@ -14,7 +17,11 @@ MODULE_SRCS += $(LOCAL_DIR)/uart.c
 
 #ROMBASE ?= 0x20400000 # if running from rom, start here
 MEMBASE ?= 0x80000000
-MEMSIZE ?= 0x00100000 # default to 1MB
+MEMSIZE ?= 0x01000000 # default to 16MB
+ifeq ($(RISCV_MODE),supervisor)
+# offset the kernel to account for OpenSBI using the bottom
+KERNEL_LOAD_OFFSET ?= 0x00200000 # kernel load offset
+endif
 
 # sifive_e or _u?
 GLOBAL_DEFINES += PLATFORM_${VARIANT}=1

project/qemu-virt-riscv64-supervisor-test.mk

@@ -0,0 +1,11 @@
+# main project for qemu-riscv64-supervisor
+MODULES += \
+	app/shell
+SUBARCH := 64
+RISCV_MODE := supervisor
+
+include project/virtual/test.mk
+include project/virtual/fs.mk
+include project/virtual/minip.mk
+include project/target/qemu-virt-riscv.mk
+

scripts/do-qemuriscv

@@ -12,6 +12,8 @@ function HELP {
     echo "-d a virtio display"
     echo "-e embeded platform"
     echo "-6 64bit"
+    echo "-u supervisor mode (using OpenSBI)"
+    echo "   currently only works in 64bit mode"
     echo "-m <memory in MB>"
     echo "-s <number of cpus>"
     echo "-h for help"
@@ -27,12 +29,14 @@ DO_EMBEDDED=0
 DO_DISPLAY=0
 DO_CMPCTMALLOC=0
 DO_MINIHEAP=0
+DO_SUPERVISOR=0
 SMP=1
 MEMSIZE=512
 SUDO=""
 PROJECT=""
+BIOS="none"
 
-while getopts bdhm:cMnte6p:s: FLAG; do
+while getopts bdhm:cMmnte6p:s:u FLAG; do
     case $FLAG in
         b) DO_BLOCK=1;;
         c) DO_CMPCTMALLOC=1;;
@@ -45,6 +49,7 @@ while getopts bdhm:cMnte6p:s: FLAG; do
         m) MEMSIZE=$OPTARG;;
         s) SMP=$OPTARG;;
         p) PROJECT=$OPTARG;;
+        u) DO_SUPERVISOR=1;;
         h) HELP;;
         \?)
             echo unrecognized option
@@ -56,16 +61,21 @@ shift $((OPTIND-1))
 
 if (( $DO_64BIT )); then
     QEMU="qemu-system-riscv64"
-    CPU="any"
+    CPU="rv64"
     MACHINE="virt"
-    _PROJECT="qemu-virt-riscv64-test"
+    if (( $DO_SUPERVISOR )); then
+        _PROJECT="qemu-virt-riscv64-supervisor-test"
+        BIOS="default"
+    else
+        _PROJECT="qemu-virt-riscv64-test"
+    fi
 elif (( $DO_EMBEDDED == 1 )); then
     QEMU="qemu-system-riscv32"
     MACHINE="sifive_e"
     _PROJECT="sifive-e-test"
 else
     QEMU="qemu-system-riscv32"
-    CPU="any"
+    CPU="rv32"
     MACHINE="virt"
     _PROJECT="qemu-virt-riscv32-test"
 fi
@@ -82,7 +92,7 @@ DISPLAY_ARGS=" -device virtio-gpu-device -serial stdio"
 # the following args only really make sense on non embedded versions
 if (( ! $DO_EMBEDDED )); then
     ARGS=" -cpu $CPU -m $MEMSIZE -smp $SMP -machine $MACHINE -kernel build-${PROJECT}/lk.elf"
-    ARGS+=" -bios none"
+    ARGS+=" -bios $BIOS"
     if (( $DO_BLOCK )); then
         ARGS+=$BLOCK_ARGS
     fi