[riscv] add a max HART define to deal with offset hart numbering

Add a define that sets the maximum allowed hart number, potentially
higher than the maximum number of allowed cpus.

This lets us more cleanly deal with having a higher HART number than the
logical cpu numbering. Only really works where it's still fairly packed
around 0, but in the case of the Sifive Unleased board it's just offset
by 1 so it's not a huge loss.

Generally clean up RISCV SMP boot code by rearranging things a bit as
well.

arch/riscv/arch.c

@@ -16,14 +16,12 @@
 #include <lk/main.h>
 #include <platform.h>
 
+#include "riscv_priv.h"
+
 #define LOCAL_TRACE 0
 
-#if WITH_SMP
-static spin_lock_t boot_cpu_lock = 1;
-volatile int secondaries_to_init = SMP_MAX_CPUS - 1;
-#endif
-
-void arch_early_init(void) {
+// first C level code to initialize each cpu
+void riscv_early_init_percpu(void) {
     // set the top level exception handler
     riscv_csr_write(RISCV_CSR_XTVEC, (uintptr_t)&riscv_exception_entry);
 
@@ -35,7 +33,26 @@ void arch_early_init(void) {
     //riscv_csr_set(mcounteren, 1);
 }
 
+// called very early just after entering C code on boot processor
+void arch_early_init(void) {
+    riscv_early_init_percpu();
+}
+
+// later init per cpu
+void riscv_init_percpu(void) {
+#if WITH_SMP
+    // enable software interrupts, used for inter-processor-interrupts
+    riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_SIE);
+#endif
+
+    // enable external interrupts
+    riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_EIE);
+}
+
+// called later once the kernel is running before platform and target init
 void arch_init(void) {
+    riscv_init_percpu();
+
     // print some arch info
     dprintf(INFO, "RISCV: mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n",
             riscv_get_mvendorid(), riscv_get_marchid(),
@@ -46,64 +63,11 @@ void arch_init(void) {
     dprintf(INFO, "RISCV: sbi %#lx (%#lx)\n", sbi_call(SBI_GET_SBI_IMPL_ID).value, sbi_call(SBI_GET_SBI_IMPL_VERSION).value);
 #endif
 
-    // enable external interrupts
-    riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_EIE);
-
 #if WITH_SMP
-    arch_mp_init_percpu();
-
-    lk_init_secondary_cpus(secondaries_to_init);
-
-    LTRACEF("RISCV: Waiting for %d secondary harts to come up\n", secondaries_to_init);
-    /* release the secondary cpus */
-    spin_unlock(&boot_cpu_lock);
-    // while (secondaries_to_init) arch_idle();
-    // spin_lock(&boot_cpu_lock);
+    riscv_boot_secondaries();
 #endif
 }
 
-#if WITH_SMP
-void riscv_secondary_entry(void) {
-    arch_early_init();
-
-    if (unlikely(arch_curr_cpu_num() >= SMP_MAX_CPUS)) {
-        while (1) {
-            arch_idle();
-        }
-    }
-
-    spin_lock(&boot_cpu_lock);
-    spin_unlock(&boot_cpu_lock);
-
-    // enable external interrupts
-    riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_EIE);
-
-    /* run early secondary cpu init routines up to the threading level */
-    lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
-
-    arch_mp_init_percpu();
-
-# if RISCV_M_MODE
-    dprintf(INFO, "RISCV: secondary hart coming up: mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n",
-            riscv_csr_read(mvendorid), riscv_csr_read(marchid),
-            riscv_csr_read(mimpid), riscv_current_hart());
-# endif /* RISCV_M_MODE */
-
-    // atomic_add(&secondaries_to_init, -1);
-    // arch_mp_send_ipi(1 << 0, MP_IPI_GENERIC); // wake up hart0 to let it know this CPU has come up
-
-    lk_secondary_cpu_entry();
-}
-
-// platform can detect and set the number of cores to boot (optional)
-void riscv_set_secondary_count(int count) {
-    if (count > SMP_MAX_CPUS - 1) {
-        count = SMP_MAX_CPUS - 1;
-    }
-    secondaries_to_init = count;
-}
-#endif
-
 void arch_idle(void) {
     // let the platform/target disable wfi
 #if !RISCV_DISABLE_WFI

arch/riscv/include/arch/arch_ops.h

@@ -63,17 +63,12 @@ static inline uint32_t arch_cycle_count(void) {
 
 static inline uint arch_curr_cpu_num(void) {
 #if WITH_SMP
-    const uint hart = riscv_current_hart();
-    for (size_t i = 0; i < SMP_MAX_CPUS; i++) {
-        if (hart_cpu_map[i] == (int)hart)
-            return i;
-        else if (unlikely(hart_cpu_map[i] == -1)) {
-            if (i != 0 || hart == BOOT_HART) {
-                hart_cpu_map[i] = hart;
-                return i;
-            }
-        }
+    uint hart = riscv_current_hart();
+    int cpu = hart_to_cpu_map[hart];
+    if (likely(cpu >= 0)) {
+        return cpu;
     }
+    panic("hart %u not assigned a cpu\n", hart);
     return -1;
 #else
     return 0;

arch/riscv/include/arch/riscv.h

@@ -120,9 +120,12 @@
     __val; \
 })
 
-extern int hart_cpu_map[SMP_MAX_CPUS];
+extern int cpu_to_hart_map[];
+extern int hart_to_cpu_map[];
+
 void riscv_set_secondary_count(int count);
 
 void riscv_exception_entry(void);
 enum handler_return riscv_timer_exception(void);
+
 #endif /* ASSEMBLY */

arch/riscv/mp.c

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2019 Elliot Berman
+ * Copyright (c) 2020 Travis Geiselbrecht
  *
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file or at
@@ -7,22 +8,48 @@
  */
 
 #include <lk/reg.h>
+#include <lk/compiler.h>
 #include <lk/debug.h>
 #include <lk/trace.h>
 #include <lk/err.h>
+#include <lk/init.h>
+#include <lk/main.h>
 
 #include <arch/ops.h>
 #include <arch/mp.h>
 #include <arch/riscv/clint.h>
 
+#include "riscv_priv.h"
+
 #if WITH_SMP
 
 #define LOCAL_TRACE 0
 
-int hart_cpu_map[SMP_MAX_CPUS] = { [0 ... SMP_MAX_CPUS-1] = -1 };
+// Highest supported HART has to at least be more than number of
+// cpus we support. Generally they're the same, but some cpus may start
+// at nonzero hart ids.
+STATIC_ASSERT(RISCV_MAX_HARTS >= SMP_MAX_CPUS);
 
-// bitmap of IPIs queued per cpu
-static volatile int ipi_data[SMP_MAX_CPUS];
+// boot hart has to be one of the valid ones
+STATIC_ASSERT(RISCV_BOOT_HART < RISCV_MAX_HARTS);
+
+// mapping of cpu -> hart
+int cpu_to_hart_map[SMP_MAX_CPUS] = {
+    [0 ... SMP_MAX_CPUS-1] = -1, // other hart cpus are assigned dynamically
+    [0] = RISCV_BOOT_HART,       // boot cpu is always logical 0
+};
+
+// mapping of hart -> cpu
+int hart_to_cpu_map[RISCV_MAX_HARTS] = {
+    [0 ... RISCV_MAX_HARTS-1] = -1,
+    [RISCV_BOOT_HART] = 0,       // boot hart is cpu 0
+};
+
+// list of IPIs queued per cpu
+static volatile int ipi_data[RISCV_MAX_HARTS];
+
+static spin_lock_t boot_cpu_lock = 1;
+volatile int secondaries_to_init = SMP_MAX_CPUS - 1;
 
 status_t arch_mp_send_ipi(mp_cpu_mask_t target, mp_ipi_t ipi) {
     LTRACEF("target 0x%x, ipi %u\n", target, ipi);
@@ -30,12 +57,16 @@ status_t arch_mp_send_ipi(mp_cpu_mask_t target, mp_ipi_t ipi) {
     mp_cpu_mask_t m = target;
     ulong hart_mask = 0;
     for (uint c = 0; c < SMP_MAX_CPUS && m; c++, m >>= 1) {
-        int h = hart_cpu_map[c];
         if (m & 1) {
+            int h = cpu_to_hart_map[c];
+            LTRACEF("c %u h %d m %#x\n", c, h, m);
+
+            // record a pending hart to notify
             hart_mask |= (1ul << h);
+
+            // set the ipi_data based on the incoming ipi
+            atomic_or(&ipi_data[h], (1u << ipi));
         }
-        // set the ipi_data based on the incoming ipi
-        atomic_or(&ipi_data[c], (1u << ipi));
     }
 
     mb();
@@ -59,7 +90,7 @@ enum handler_return riscv_software_exception(void) {
 
     rmb();
     int reason = atomic_swap(&ipi_data[ch], 0);
-    LTRACEF("reason %#x\n", reason);
+    LTRACEF("ch %u reason %#x\n", ch, reason);
 
     enum handler_return ret = INT_NO_RESCHEDULE;
     if (reason & (1u << MP_IPI_RESCHEDULE)) {
@@ -79,9 +110,66 @@ enum handler_return riscv_software_exception(void) {
     return ret;
 }
 
-void arch_mp_init_percpu(void) {
-    dprintf(INFO, "RISCV: Booting hart%d (cpu%d)\n", riscv_current_hart(), arch_curr_cpu_num());
-    riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_SIE);
+void riscv_secondary_entry(int cpu_id) {
+    // basic bootstrapping of this cpu
+    riscv_early_init_percpu();
+
+    // assign secondary cpu an id, starting at cpu 1
+    // cpu 0 is always the boot hart
+    static volatile int secondary_cpu_id = 1;
+    int myid = atomic_add(&secondary_cpu_id, 1);
+    uint hart = riscv_current_hart();
+    cpu_to_hart_map[myid] = hart;
+    hart_to_cpu_map[hart] = myid;
+    wmb();
+
+    if (unlikely(arch_curr_cpu_num() >= SMP_MAX_CPUS)) {
+        while (1) {
+            arch_idle();
+        }
+    }
+
+    spin_lock(&boot_cpu_lock);
+    spin_unlock(&boot_cpu_lock);
+
+    // run early secondary cpu init routines up to the threading level
+    lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
+
+    // run threading level initialization on this cpu
+    riscv_init_percpu();
+
+    dprintf(INFO, "RISCV: secondary hart coming up: mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n",
+            riscv_get_mvendorid(), riscv_get_marchid(),
+            riscv_get_mimpid(), riscv_current_hart());
+
+    // atomic_add(&secondaries_to_init, -1);
+    // arch_mp_send_ipi(1 << 0, MP_IPI_GENERIC); // wake up hart0 to let it know this CPU has come up
+
+    lk_secondary_cpu_entry();
 }
 
+// platform can detect and set the number of cores to boot (optional)
+void riscv_set_secondary_count(int count) {
+    if (count > SMP_MAX_CPUS - 1) {
+        count = SMP_MAX_CPUS - 1;
+    }
+    secondaries_to_init = count;
+}
+
+// start any secondary cpus we are set to start. called on the boot processor
+void riscv_boot_secondaries(void) {
+    lk_init_secondary_cpus(secondaries_to_init);
+
+    LTRACEF("RISCV: Waiting for %d secondary harts to come up\n", secondaries_to_init);
+    /* release the secondary cpus */
+    spin_unlock(&boot_cpu_lock);
+
+    // wait a second while the secondaries start
+    //spin(1000000);
+
+    // while (secondaries_to_init) arch_idle();
+    // spin_lock(&boot_cpu_lock);
+}
+
+
 #endif

arch/riscv/riscv_priv.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2020 Travis Geiselbrecht
+ *
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT
+ */
+#pragma once
+
+void riscv_early_init_percpu(void);
+void riscv_init_percpu(void);
+void riscv_boot_secondaries(void);
+

arch/riscv/rules.mk

@@ -10,12 +10,17 @@ MODULE_SRCS += $(LOCAL_DIR)/thread.c
 MODULE_SRCS += $(LOCAL_DIR)/mp.c
 MODULE_SRCS += $(LOCAL_DIR)/time.c
 
+# one file uses slightly complicated designated initializer
+MODULE_CFLAGS += -Wno-override-init
+
 SMP_MAX_CPUS ?= 1
-BOOT_HART ?= 0
+RISCV_MAX_HARTS ?= $(SMP_MAX_CPUS)
+RISCV_BOOT_HART ?= 0
 
 GLOBAL_DEFINES += SMP_MAX_CPUS=$(SMP_MAX_CPUS)
+GLOBAL_DEFINES += RISCV_MAX_HARTS=$(RISCV_MAX_HARTS)
+GLOBAL_DEFINES += RISCV_BOOT_HART=$(RISCV_BOOT_HART)
 GLOBAL_DEFINES += PLATFORM_HAS_DYNAMIC_TIMER=1
-GLOBAL_DEFINES += BOOT_HART=$(BOOT_HART)
 
 ifeq ($(WITH_SMP),1)
 GLOBAL_DEFINES += WITH_SMP=1

arch/riscv/start.S

@@ -23,7 +23,11 @@ FUNCTION(_start)
     csrw    sscratch, a0
 #endif
 
-    // set the default stack
+    // if the hart is too high, trap it
+    li      t0, RISCV_MAX_HARTS
+    ble     t0, a0, .Lhart_trap
+
+    // set the default stack per cpu
     la      sp, default_stack_top
     // default stack locations for each hart:
     // LOW ------------ HIGH
@@ -32,8 +36,8 @@ FUNCTION(_start)
     mul     t1, t1, a0
     sub     sp, sp, t1
 
-    // if our hart isnt BOOT_HART, trap the cpu
-    li      t2, BOOT_HART
+    // if our hart isnt RISCV_BOOT_HART, trap the cpu
+    li      t2, RISCV_BOOT_HART
     bne     t2, a0, .Lsecondary_trap
 
 #if ARCH_RISCV_TWOSEGMENT
@@ -56,16 +60,16 @@ FUNCTION(_start)
     la      t0, __bss_start
     la      t1, __bss_end
 0:
-    sw      x0, (t0)
+    sw      zero, (t0)
     add     t0, t0, 4
     bne     t0, t1, 0b
 
 #if WITH_SMP
     // Release any other harts into riscv_secondary_entry
     fence   w, w
-    la      t5, _boot_status
+    la      t1, _boot_status
     li      t0, 1
-    sb      t0, (t5)
+    sb      t0, (t1)
 #endif
 
     // call main
@@ -86,10 +90,15 @@ FUNCTION(_start)
     j       .
 #endif
 
+.Lhart_trap:
+    // cpus with too high of a hart id go here and spin forever
+    wfi
+    j       .
+
 .bss
 .align 4
 LOCAL_DATA(default_stack)
-    .skip ARCH_DEFAULT_STACK_SIZE * SMP_MAX_CPUS
+    .skip ARCH_DEFAULT_STACK_SIZE * RISCV_MAX_HARTS;
 LOCAL_DATA(default_stack_top)
 
 // put boot status in .data so it doesn't get paved over during BSS initialization