WIP [target][visionfive2] Add initial support for a VisionFive 2 RISC-V board

Still TODO:
-Set the timer rate properly
-Fix nonzero based hart secondary cpu boot
-Try to parse the device tree for some information

.github/workflows/github-ci.yml

@@ -43,6 +43,7 @@ jobs:
           - pico-test
           - sifive-e-test
           - sifive-unleashed-test
+          - visionfive2-test
           - rosco-m68k-test
         exclude:
           # no toolchain for 7.5.0 for or1k

arch/riscv/mp.c

@@ -152,16 +152,27 @@ void riscv_set_secondary_count(int count) {
 
 // start any secondary cpus we are set to start. called on the boot processor
 void riscv_boot_secondaries(void) {
+    // if theres nothing to start, abort here
+    if (secondaries_to_init == 0) {
+        dprintf(INFO, "RISCV: No secondary harts to start\n");
+        return;
+    }
+
     lk_init_secondary_cpus(secondaries_to_init);
 
 #if RISCV_M_MODE
     dprintf(INFO, "RISCV: Releasing %d secondary harts from purgatory\n", secondaries_to_init);
 #else
     uint boot_hart = riscv_current_hart();
+    dprintf(INFO, "RISCV: Going to try to start %d secondary harts\n", secondaries_to_init);
 
     // use SBI HSM to boot the secondaries
     // TODO: handle the range of harts we should consider, since they
-    // may not be zero based
+    // may not be zero based.
+    // Currently, starts from 0 and tries to start one extra core, with the idea
+    // that boot_hart should be one of them. This algorithm is somewhat broken, but
+    // works in the case of harts being 0-N and the boot hart being nonzero (but within [0...N]).
+    // Doesn't currently handle skipping cpus we shouldn't boot (like HART 0 on some machines)
     for (uint i = 0; i <= (uint)secondaries_to_init; i++) {
         // skip the boot hart
         if (i != boot_hart) {

platform/jh7110/include/platform/jh7110.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2018 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#pragma once
+
+// memory and irq layout of JH7110
+#define MEMORY_BASE_PHYS     (0x40000000)
+// up to 16 GB of ram
+#define MEMORY_APERTURE_SIZE (16ULL * 1024 * 1024 * 1024)
+
+// map all of 0-1GB into kernel space in one shot
+#define PERIPHERAL_BASE_PHYS (0)
+#define PERIPHERAL_BASE_SIZE (0x40000000UL) // 1GB
+
+// use the giant mapping at the bottom of the kernel as our peripheral space
+#define PERIPHERAL_BASE_VIRT (KERNEL_ASPACE_BASE + PERIPHERAL_BASE_PHYS)
+
+// interrupts
+#define IRQ_VIRTIO_BASE 1
+#define IRQ_UART0       0x20
+#define NUM_IRQS        127
+
+// addresses of some peripherals
+#define CLINT_BASE          0x02000000
+#define CLINT_BASE_VIRT     (PERIPHERAL_BASE_VIRT + CLINT_BASE)
+#define PLIC_BASE           0x0c000000
+#define PLIC_BASE_VIRT      (PERIPHERAL_BASE_VIRT + PLIC_BASE)
+#define UART0_BASE          0x10000000
+#define UART0_BASE_VIRT     (PERIPHERAL_BASE_VIRT + UART0_BASE)
+#define DRAM_BASE           0x40000000
+#define DRAM_BASE_VIRT      (PERIPHERAL_BASE_VIRT + DRAM_BASE)

platform/jh7110/platform.c

@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2018 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#include <inttypes.h>
+#include <lk/err.h>
+#include <lk/reg.h>
+#include <lk/trace.h>
+#include <kernel/thread.h>
+#include <platform.h>
+#include <platform/interrupts.h>
+#include <platform/debug.h>
+#include <platform/timer.h>
+#include <platform/jh7110.h>
+#include <sys/types.h>
+#include <lib/fdtwalk.h>
+#if WITH_LIB_MINIP
+#include <lib/minip.h>
+#endif
+#include <kernel/vm.h>
+#if WITH_LIB_CONSOLE
+#include <lib/console.h>
+#endif
+
+#include "platform_p.h"
+
+#define LOCAL_TRACE 1
+
+extern ulong lk_boot_args[4];
+
+#if WITH_KERNEL_VM
+#define DEFAULT_MEMORY_SIZE (MEMSIZE) /* try to fetch from the emulator via the fdt */
+
+static pmm_arena_t arena = {
+    .name = "ram",
+    .base = MEMORY_BASE_PHYS,
+    .size = DEFAULT_MEMORY_SIZE,
+    .flags = PMM_ARENA_FLAG_KMAP,
+};
+#endif
+
+static volatile uint32_t *power_reset_reg;
+
+// callbacks to the fdt_walk routine
+static void memcallback(uint64_t base, uint64_t len, void *cookie) {
+    bool *found_mem = (bool *)cookie;
+
+    LTRACEF("base %#llx len %#llx cookie %p\n", base, len, cookie);
+
+    /* add another vm arena */
+    if (!*found_mem) {
+        printf("FDT: found memory arena, base %#llx size %#llx\n", base, len);
+#if WITH_KERNEL_VM
+        arena.base = base;
+        arena.size = len;
+        pmm_add_arena(&arena);
+#else
+        novm_add_arena("fdt", base, len);
+#endif
+        *found_mem = true; // stop searching after the first one
+    }
+}
+
+struct reserved_memory_regions {
+    size_t count;
+
+    struct {
+        uint64_t base;
+        uint64_t len;
+    } regions[16];
+};
+
+static void reserved_memory_callback(uint64_t base, uint64_t len, void *cookie) {
+    struct reserved_memory_regions *mem = cookie;
+
+    LTRACEF("base %#llx len %#llx\n", base, len);
+
+    if (mem->count < countof(mem->regions)) {
+        mem->regions[mem->count].base = base;
+        mem->regions[mem->count].len = len;
+        mem->count++;
+    }
+}
+
+static void cpucallback(uint64_t id, void *cookie) {
+    int *cpu_count = (int *)cookie;
+
+    LTRACEF("id %#llx cookie %p\n", id, cookie);
+
+    (*cpu_count)++;
+}
+
+struct pcie_detect_state {
+    struct fdt_walk_pcie_info info;
+} pcie_state;
+
+static void pciecallback(const struct fdt_walk_pcie_info *info, void *cookie) {
+    struct pcie_detect_state *state = cookie;
+
+    LTRACEF("ecam base %#llx, len %#llx, bus_start %hhu, bus_end %hhu\n", info->ecam_base, info->ecam_len, info->bus_start, info->bus_end);
+    state->info = *info;
+}
+
+void platform_early_init(void) {
+    TRACE;
+    plic_early_init();
+
+    LTRACEF("starting FDT scan\n");
+
+    /* look for a flattened device tree in the second arg passed to us */
+    bool found_mem = false;
+    int cpu_count = 0;
+    struct reserved_memory_regions reserved = {0};
+
+    const void *fdt = (void *)lk_boot_args[1];
+#if WITH_KERNEL_VM
+    fdt = (const void *)((uintptr_t)fdt + PERIPHERAL_BASE_VIRT);
+#endif
+
+    struct fdt_walk_callbacks cb = {
+        .mem = memcallback,
+        .memcookie = &found_mem,
+        .reserved_memory = reserved_memory_callback,
+        .reserved_memory_cookie = &reserved,
+        .cpu = cpucallback,
+        .cpucookie = &cpu_count,
+        .pcie = pciecallback,
+        .pciecookie = &pcie_state,
+    };
+
+    status_t err = fdt_walk(fdt, &cb);
+    LTRACEF("fdt_walk returns %d\n", err);
+
+    if (err != 0) {
+        printf("FDT: error finding FDT at %p, using default memory & cpu count\n", fdt);
+        reserved.regions[0].base = MEMBASE;
+        reserved.regions[0].len = 0x00200000; // reserve the first 2MB of memory for SBI
+        reserved.count = 1;
+    }
+
+    /* add a default memory region if we didn't find it in the FDT */
+    if (!found_mem) {
+#if WITH_KERNEL_VM
+        pmm_add_arena(&arena);
+#else
+        novm_add_arena("default", MEMBASE, MEMSIZE);
+#endif
+    }
+
+#if WITH_KERNEL_VM
+    /* reserve memory described by the FDT */
+    for (size_t i = 0; i < reserved.count; i++) {
+        printf("FDT: reserving memory range [%#llx ... %#llx]\n",
+                reserved.regions[i].base, reserved.regions[i].base + reserved.regions[i].len - 1);
+        struct list_node list = LIST_INITIAL_VALUE(list);
+        pmm_alloc_range(reserved.regions[i].base, reserved.regions[i].len / PAGE_SIZE, &list);
+    }
+#endif
+
+    if (cpu_count > 0) {
+        printf("FDT: found %d cpu%c\n", cpu_count, cpu_count == 1 ? ' ' : 's');
+        riscv_set_secondary_count(cpu_count - 1);
+    } else {
+        riscv_set_secondary_count(0);
+    }
+
+    LTRACEF("done scanning FDT\n");
+
+    /* save a copy of the pointer to the poweroff/reset register */
+    /* TODO: read it from the FDT */
+#if WITH_KERNEL_VM
+    power_reset_reg = paddr_to_kvaddr(0x100000);
+#else
+    power_reset_reg = (void *)0x100000;
+#endif
+}
+
+void platform_init(void) {
+    plic_init();
+    uart_init();
+}
+
+void platform_halt(platform_halt_action suggested_action,
+                          platform_halt_reason reason) {
+    switch (suggested_action) {
+        case HALT_ACTION_SHUTDOWN:
+            dprintf(ALWAYS, "Shutting down... (reason = %d)\n", reason);
+#if RISCV_S_MODE
+            // try to use SBI as a cleaner way to stop
+            sbi_system_reset(SBI_RESET_TYPE_SHUTDOWN, SBI_RESET_REASON_NONE);
+#endif
+            *power_reset_reg = 0x5555;
+            break;
+        case HALT_ACTION_REBOOT:
+            dprintf(ALWAYS, "Rebooting... (reason = %d)\n", reason);
+#if RISCV_S_MODE
+            sbi_system_reset(SBI_RESET_TYPE_COLD_REBOOT, SBI_RESET_REASON_NONE);
+#endif
+            *power_reset_reg = 0x7777;
+            break;
+        case HALT_ACTION_HALT:
+#if ENABLE_PANIC_SHELL
+            if (reason == HALT_REASON_SW_PANIC) {
+                dprintf(ALWAYS, "CRASH: starting debug shell... (reason = %d)\n", reason);
+                arch_disable_ints();
+                panic_shell_start();
+            }
+#endif  // ENABLE_PANIC_SHELL
+            dprintf(ALWAYS, "HALT: spinning forever... (reason = %d)\n", reason);
+            break;
+    }
+
+    arch_disable_ints();
+    for (;;)
+        arch_idle();
+}

platform/jh7110/platform_p.h

@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2018 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#pragma once
+
+#include <stdbool.h>
+
+void uart_init(void);
+
+void plic_early_init(void);
+void plic_init(void);
+
+

platform/jh7110/plic.c

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2018 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#include "platform_p.h"
+
+#include <assert.h>
+#include <lk/err.h>
+#include <lk/debug.h>
+#include <lk/reg.h>
+#include <lk/trace.h>
+#include <kernel/debug.h>
+#include <kernel/thread.h>
+#include <platform/interrupts.h>
+#include <platform/jh7110.h>
+
+#define LOCAL_TRACE 0
+
+// Driver for PLIC implementation for qemu riscv virt machine
+#define PLIC_PRIORITY(irq) (PLIC_BASE_VIRT + 4 * (irq))
+#define PLIC_PENDING(irq)  (PLIC_BASE_VIRT + 0x1000 + (4 * ((irq) / 32)))
+#define PLIC_ENABLE(irq, hart)      (PLIC_BASE_VIRT + 0x2000 + (0x80 * plic_hart_index(hart)) + (4 * ((irq) / 32)))
+#define PLIC_THRESHOLD(hart)        (PLIC_BASE_VIRT + 0x200000 + (0x1000 * plic_hart_index(hart)))
+#define PLIC_COMPLETE(hart)         (PLIC_BASE_VIRT + 0x200004 + (0x1000 * plic_hart_index(hart)))
+#define PLIC_CLAIM(hart)            PLIC_COMPLETE(hart)
+
+static struct int_handlers {
+    int_handler handler;
+    void *arg;
+} handlers[NUM_IRQS];
+
+// Mapping of HART to interrupt target is annoyingly complex:
+// On the JH7110 (like other sifive socs) the first HART only has one mode, machine
+// and the subsequent harts have both machine and supervisor. The interrupt targets
+// are thus indexed:
+// HART 0 machine mode = 0
+// HART 1 machine mode = 1
+// HART 1 supervisor mode = 2
+// HART 2 machine mode = 3
+// HART 2 supervisor mode = 4
+// ...
+//
+// This routine maps harts to the current mode's interrupt target
+static unsigned int plic_hart_index(unsigned int hart) {
+    unsigned int index;
+#if RISCV_M_MODE
+    index = (hart == 0) ? 0 : (2 * hart - 1);
+#elif RISCV_S_MODE
+    DEBUG_ASSERT(hart != 0);
+    index = 2 * hart;
+#else
+#error undefined
+#endif
+    return index;
+}
+
+void plic_early_init(void) {
+    // mask all irqs and set their priority to 1
+    // TODO: mask on all the other cpus too
+    for (int i = 1; i < NUM_IRQS; i++) {
+        *REG32(PLIC_ENABLE(i, riscv_current_hart())) &= ~(1 << (i % 32));
+        *REG32(PLIC_PRIORITY(i)) = 1;
+    }
+
+    // set global priority threshold to 0
+    *REG32(PLIC_THRESHOLD(riscv_current_hart())) = 0;
+}
+
+void plic_init(void) {
+}
+
+status_t mask_interrupt(unsigned int vector) {
+    LTRACEF("vector %u, current hart %u\n", vector, riscv_current_hart());
+    *REG32(PLIC_ENABLE(vector, riscv_current_hart())) &= ~(1 << (vector % 32));
+    return NO_ERROR;
+}
+
+status_t unmask_interrupt(unsigned int vector) {
+    LTRACEF("vector %u, current hart %u\n", vector, riscv_current_hart());
+    *REG32(PLIC_ENABLE(vector, riscv_current_hart())) |= (1 << (vector % 32));
+
+    return NO_ERROR;
+}
+
+void register_int_handler(unsigned int vector, int_handler handler, void *arg) {
+    LTRACEF("vector %u handler %p arg %p, hart %u\n", vector, handler, arg, riscv_current_hart());
+
+    DEBUG_ASSERT(vector < NUM_IRQS);
+
+    handlers[vector].handler = handler;
+    handlers[vector].arg = arg;
+}
+
+void register_int_handler_msi(unsigned int vector, int_handler handler, void *arg, bool edge) {
+    PANIC_UNIMPLEMENTED;
+}
+
+enum handler_return riscv_platform_irq(void) {
+    // see what irq triggered it
+    uint32_t vector = *REG32(PLIC_CLAIM(riscv_current_hart()));
+    LTRACEF("vector %u\n", vector);
+
+    if (unlikely(vector == 0)) {
+        // nothing pending
+        return INT_NO_RESCHEDULE;
+    }
+
+    THREAD_STATS_INC(interrupts);
+    KEVLOG_IRQ_ENTER(vector);
+
+    enum handler_return ret = INT_NO_RESCHEDULE;
+    if (handlers[vector].handler) {
+        ret = handlers[vector].handler(handlers[vector].arg);
+    }
+
+    // ack the interrupt
+    *REG32(PLIC_COMPLETE(riscv_current_hart())) = vector;
+
+    KEVLOG_IRQ_EXIT(vector);
+
+    return ret;
+}
+
+status_t platform_pci_int_to_vector(unsigned int pci_int, unsigned int *vector) {
+    // at the moment there's no translation between PCI IRQs and native irqs
+    *vector = pci_int;
+    return NO_ERROR;
+}
+
+status_t platform_allocate_interrupts(size_t count, uint align_log2, bool msi, unsigned int *vector) {
+    return ERR_NOT_SUPPORTED;
+}
+
+status_t platform_compute_msi_values(unsigned int vector, unsigned int cpu, bool edge,
+        uint64_t *msi_address_out, uint16_t *msi_data_out) {
+    return ERR_NOT_SUPPORTED;
+}

platform/jh7110/rules.mk

@@ -0,0 +1,41 @@
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+MODULE := $(LOCAL_DIR)
+
+ARCH := riscv
+SUBARCH ?= 64
+RISCV_MODE ?= supervisor
+WITH_SMP ?= true
+SMP_MAX_CPUS ?= 4
+LK_HEAP_IMPLEMENTATION ?= dlmalloc
+RISCV_FPU ?= true
+RISCV_MMU ?= sv39
+
+MODULE_DEPS += lib/cbuf
+MODULE_DEPS += lib/fdt
+MODULE_DEPS += lib/fdtwalk
+#MODULE_DEPS += dev/bus/pci
+#MODULE_DEPS += dev/bus/pci/drivers
+
+MODULE_SRCS += $(LOCAL_DIR)/platform.c
+MODULE_SRCS += $(LOCAL_DIR)/plic.c
+MODULE_SRCS += $(LOCAL_DIR)/uart.c
+
+MEMBASE ?= 0x40000000
+MEMSIZE ?= 0x10000000 # default to 256MB
+ifeq ($(RISCV_MODE),supervisor)
+# offset the kernel to account for OpenSBI using the bottom
+KERNEL_LOAD_OFFSET ?= 0x00200000 # kernel load offset
+endif
+
+# set some global defines based on capability
+GLOBAL_DEFINES += ARCH_RISCV_CLINT_BASE=0x0c000000
+GLOBAL_DEFINES += ARCH_RISCV_MTIME_RATE=10000000
+
+# we can revert to a poll based uart spin routine
+GLOBAL_DEFINES += PLATFORM_SUPPORTS_PANIC_SHELL=1
+
+# do not need to implement any cache ops
+GLOBAL_DEFINES += RISCV_NO_CACHE_OPS=1
+
+include make/module.mk

platform/jh7110/uart.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2018 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#include <lk/reg.h>
+#include <lk/trace.h>
+#include <lib/cbuf.h>
+#include <kernel/thread.h>
+#include <platform.h>
+#include <platform/interrupts.h>
+#include <platform/debug.h>
+#include <platform/jh7110.h>
+#include <sys/types.h>
+
+#include "platform_p.h"
+
+// simple 16550 driver for the emulated serial port on jh7110
+// uart registers are 4 byte separated
+
+static volatile uint8_t *const uart_base = (uint8_t *)UART0_BASE_VIRT;
+
+#define RXBUF_SIZE 128
+static char uart_rx_buf_data[RXBUF_SIZE];
+static cbuf_t uart_rx_buf;
+
+static inline uint8_t uart_read_8(size_t offset) {
+    return uart_base[offset * 4];
+}
+
+static inline void uart_write_8(size_t offset, uint8_t val) {
+    uart_base[offset * 4] = val;
+}
+
+static enum handler_return uart_irq_handler(void *arg) {
+    unsigned char c;
+    bool resched = false;
+
+    while (uart_read_8(5) & (1<<0)) {
+        c = uart_read_8(0);
+        cbuf_write_char(&uart_rx_buf, c, false);
+        resched = true;
+    }
+
+    return resched ? INT_RESCHEDULE : INT_NO_RESCHEDULE;
+}
+
+void uart_init(void) {
+    /* finish uart init to get rx going */
+    cbuf_initialize_etc(&uart_rx_buf, RXBUF_SIZE, uart_rx_buf_data);
+
+    register_int_handler(IRQ_UART0, uart_irq_handler, NULL);
+
+    uart_write_8(1, 0x1); // enable receive data available interrupt
+
+    unmask_interrupt(IRQ_UART0);
+}
+
+static void uart_putc(char c) {
+    while ((uart_read_8(5) & (1<<6)) == 0)
+        ;
+    uart_write_8(0, c);
+}
+
+static int uart_getc(char *c, bool wait) {
+    return cbuf_read_char(&uart_rx_buf, c, wait);
+}
+
+void platform_dputc(char c) {
+    if (c == '\n')
+        platform_dputc('\r');
+    uart_putc(c);
+}
+
+int platform_dgetc(char *c, bool wait) {
+    int ret = uart_getc(c, wait);
+
+    return ret;
+}
+
+/* panic-time getc/putc */
+int platform_pgetc(char *c, bool wait) {
+    if (uart_read_8(5) & (1<<0)) {
+        *c = uart_read_8(0);
+        return 0;
+    }
+    return -1;
+}
+

project/target/visionfive2.mk

@@ -0,0 +1,2 @@
+TARGET := visionfive2
+

project/visionfive2-test.mk

@@ -0,0 +1,7 @@
+# main project for visionfive 2 test project
+MODULES += \
+	app/shell
+
+include project/virtual/test.mk
+include project/target/visionfive2.mk
+

target/visionfive2/rules.mk

@@ -0,0 +1,7 @@
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+MODULE := $(LOCAL_DIR)
+
+PLATFORM := jh7110
+
+MEMSIZE ?= 0x200000000 # 8GB