[target][sifive-e] Get LK working on a Sifive Hifive1 board

-added support for bringing up the clocks and setting up the gpio bits
before starting the uart.
-add a proper target init routine
-add scripts to flash board via openocd
-fixed bug in riscv interrupt save state where it wasn't saving mstatus
on irq entry.
-comment out cycle enabling, not implemented on this core

arch/riscv/arch.c

@@ -20,10 +20,12 @@
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
+#include <assert.h>
 #include <trace.h>
 #include <debug.h>
 #include <stdint.h>
 #include <arch/riscv.h>
+#include <arch/ops.h>
 
 #define LOCAL_TRACE 0
 
@@ -35,17 +37,24 @@ void arch_early_init(void) {
     riscv_csr_clear(mstatus, RISCV_STATUS_MIE);
     riscv_csr_clear(mie, RISCV_MIE_MTIE | RISCV_MIE_MSIE | RISCV_MIE_SEIE | RISCV_MIE_MEIE);
 
-    // enable cycle counter
-    riscv_csr_set(mcounteren, 1);
+    // enable cycle counter (disabled for now, unimplemented on sifive-e)
+    //riscv_csr_set(mcounteren, 1);
 }
 
 void arch_init(void) {
+    // print some arch info
+    dprintf(INFO, "RISCV: mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#lx\n",
+            riscv_csr_read(mvendorid), riscv_csr_read(marchid),
+            riscv_csr_read(mimpid), riscv_csr_read(mhartid));
+    dprintf(INFO, "RISCV: misa %#lx\n", riscv_csr_read(misa));
+
     // enable external interrupts
     riscv_csr_set(mie, RISCV_MIE_MEIE);
 }
 
 void arch_idle(void) {
     // disabled for now, QEMU seems to have some trouble emulating wfi properly
+    // also have trouble breaking into sifive-e board with openocd when wfi
 //    __asm__ volatile("wfi");
 }
 

arch/riscv/asm.S

@@ -67,22 +67,24 @@ FUNCTION(riscv32_context_switch)
 FUNCTION(riscv_exception_entry)
     /* dump all the callee trashed regs on the stack */
     addi   sp, sp, -80 // subtract a multiple of 16 to align the stack
-    sw     t6, 64(sp)
-    sw     t5, 60(sp)
-    sw     t4, 56(sp)
-    sw     t3, 52(sp)
-    sw     t2, 48(sp)
-    sw     t1, 44(sp)
-    sw     t0, 40(sp)
-    sw     a7, 36(sp)
-    sw     a6, 32(sp)
-    sw     a5, 28(sp)
-    sw     a4, 24(sp)
-    sw     a3, 20(sp)
-    sw     a2, 16(sp)
-    sw     a1, 12(sp)
-    sw     a0, 8(sp)
-    sw     ra, 4(sp)
+    sw     t6, 68(sp)
+    sw     t5, 64(sp)
+    sw     t4, 60(sp)
+    sw     t3, 56(sp)
+    sw     t2, 52(sp)
+    sw     t1, 48(sp)
+    sw     t0, 44(sp)
+    sw     a7, 40(sp)
+    sw     a6, 36(sp)
+    sw     a5, 32(sp)
+    sw     a4, 28(sp)
+    sw     a3, 24(sp)
+    sw     a2, 20(sp)
+    sw     a1, 16(sp)
+    sw     a0, 12(sp)
+    sw     ra, 8(sp)
+    csrr   t0, mstatus
+    sw     t0, 4(sp)
     csrr   a0, mcause
     csrr   a1, mepc
     sw     a1, 0(sp)
@@ -93,23 +95,25 @@ FUNCTION(riscv_exception_entry)
     /* put everything back */
     lw     t0, 0(sp)
     csrw   mepc, t0
+    lw     t0, 4(sp)
+    csrw   mstatus, t0
 
-    lw     ra, 4(sp)
-    lw     a0, 8(sp)
-    lw     a1, 12(sp)
-    lw     a2, 16(sp)
-    lw     a3, 20(sp)
-    lw     a4, 24(sp)
-    lw     a5, 28(sp)
-    lw     a6, 32(sp)
-    lw     a7, 36(sp)
-    lw     t0, 40(sp)
-    lw     t1, 44(sp)
-    lw     t2, 48(sp)
-    lw     t3, 52(sp)
-    lw     t4, 56(sp)
-    lw     t5, 60(sp)
-    lw     t6, 64(sp)
+    lw     ra, 8(sp)
+    lw     a0, 12(sp)
+    lw     a1, 16(sp)
+    lw     a2, 20(sp)
+    lw     a3, 24(sp)
+    lw     a4, 28(sp)
+    lw     a5, 32(sp)
+    lw     a6, 36(sp)
+    lw     a7, 40(sp)
+    lw     t0, 44(sp)
+    lw     t1, 48(sp)
+    lw     t2, 52(sp)
+    lw     t3, 56(sp)
+    lw     t4, 60(sp)
+    lw     t5, 64(sp)
+    lw     t6, 68(sp)
     addi   sp, sp, 80
 
     mret

arch/riscv/clint.c

@@ -44,7 +44,11 @@
 #define CLINT_MTIME (ARCH_RISCV_CLINT_BASE + 0xbff8)
 
 lk_bigtime_t current_time_hires(void) {
+#if ARCH_RISCV_MTIME_RATE < 10000000
+    return current_time() * 1000llu; // hack to deal with slow clocks
+#else
     return *REG64(CLINT_MTIME) / (ARCH_RISCV_MTIME_RATE / 1000000u);
+#endif
 }
 
 lk_time_t current_time(void) {
@@ -64,7 +68,7 @@ status_t platform_set_oneshot_timer (platform_timer_callback callback, void *arg
     timer_arg = arg;
 
     // convert interval to ticks
-    uint64_t ticks = interval * (ARCH_RISCV_MTIME_RATE / 1000u);
+    uint64_t ticks = (interval * ARCH_RISCV_MTIME_RATE) / 1000u;
     *REG64(CLINT_MTIMECMP(0)) = *REG64(CLINT_MTIME) + ticks;
 
     // enable the timer

arch/riscv/exceptions.c

@@ -20,6 +20,7 @@
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
+#include <assert.h>
 #include <compiler.h>
 #include <trace.h>
 #include <arch/riscv.h>
@@ -30,6 +31,7 @@
 // keep in sync with asm.S
 struct riscv_short_iframe {
     ulong  mepc;
+    ulong  mstatus;
     ulong  ra;
     ulong  a0;
     ulong  a1;
@@ -53,6 +55,9 @@ extern enum handler_return riscv_platform_irq(void);
 void riscv_exception_handler(ulong cause, ulong epc, struct riscv_short_iframe *frame) {
     LTRACEF("cause %#lx epc %#lx\n", cause, epc);
 
+    DEBUG_ASSERT(arch_ints_disabled());
+    DEBUG_ASSERT(frame->mstatus & RISCV_STATUS_MPIE);
+
     enum handler_return ret = INT_NO_RESCHEDULE;
     switch (cause) {
         case 0x80000007: // machine timer interrupt
@@ -62,11 +67,17 @@ void riscv_exception_handler(ulong cause, ulong epc, struct riscv_short_iframe *
             ret = riscv_platform_irq();
             break;
         default:
-            TRACEF("unhandled cause %#lx, epc %#lx\n", cause, epc);
+            TRACEF("unhandled cause %#lx, epc %#lx, mtval %#lx\n", cause, epc, riscv_csr_read(mtval));
             panic("stopping");
     }
 
+    DEBUG_ASSERT(arch_ints_disabled());
+    DEBUG_ASSERT(frame->mstatus & RISCV_STATUS_MPIE);
+
     if (ret == INT_RESCHEDULE) {
         thread_preempt();
     }
+
+    DEBUG_ASSERT(arch_ints_disabled());
+    DEBUG_ASSERT(frame->mstatus & RISCV_STATUS_MPIE);
 }

arch/riscv/include/arch/arch_ops.h

@@ -69,7 +69,8 @@ static inline void set_current_thread(struct thread *t) {
 static inline uint32_t arch_cycle_count(void) {
     uint32_t count;
 
-    __asm__("rdcycle %0" : "=r"(count));
+    //__asm__("rdcycle %0" : "=r"(count));
+    count = riscv_csr_read(mcycle);
     return count;
 }
 

arch/riscv/include/arch/riscv.h

@@ -47,6 +47,18 @@
         : "memory"); \
 })
 
+#define riscv_csr_read_clear(csr, bits) \
+({ \
+    ulong __val = bits; \
+    ulong __val_out; \
+    __asm__ volatile( \
+        "csrrc   %0, " #csr ", %1" \
+        : "=r"(__val_out) \
+        : "rK" (__val) \
+        : "memory"); \
+    __val_out; \
+})
+
 #define riscv_csr_set(csr, bits) \
 ({ \
     ulong __val = bits; \

arch/riscv/include/arch/spinlock.h

@@ -33,7 +33,7 @@
 
 typedef unsigned int spin_lock_t;
 
-typedef unsigned int spin_lock_saved_state_t;
+typedef unsigned long spin_lock_saved_state_t;
 typedef unsigned int spin_lock_save_flags_t;
 
 static inline void arch_spin_lock(spin_lock_t *lock) {
@@ -59,27 +59,15 @@ static inline bool arch_spin_lock_held(spin_lock_t *lock) {
 /* default arm flag is to just disable plain irqs */
 #define ARCH_DEFAULT_SPIN_LOCK_FLAG_INTERRUPTS  0
 
-enum {
-    /* private */
-    SPIN_LOCK_STATE_RESTORE_IRQ = 1,
-};
-
 static inline void
 arch_interrupt_save(spin_lock_saved_state_t *statep, spin_lock_save_flags_t flags) {
-    spin_lock_saved_state_t state = 0;
-    if (!arch_ints_disabled()) {
-        state |= SPIN_LOCK_STATE_RESTORE_IRQ;
-        arch_disable_ints();
-    }
-    *statep = state;
+    /* disable interrupts by clearing the MIE bit while atomically saving the old state */
+    *statep = riscv_csr_read_clear(mstatus, RISCV_STATUS_MIE) & RISCV_STATUS_MIE;
 }
 
 static inline void
 arch_interrupt_restore(spin_lock_saved_state_t old_state, spin_lock_save_flags_t flags) {
-    if (old_state & SPIN_LOCK_STATE_RESTORE_IRQ)
-        arch_enable_ints();
+    /* drop the old MIE flag into the status register */
+    riscv_csr_set(mstatus, old_state);
 }
 
-
-
-

arch/riscv/rules.mk

@@ -22,7 +22,7 @@ WITH_LINKER_GC ?= 0
 cc-option = $(shell if test -z "`$(1) $(2) -S -o /dev/null -xc /dev/null 2>&1`"; \
 	then echo "$(2)"; else echo "$(3)"; fi ;)
 
-ARCH_COMPILEFLAGS :=
+ARCH_COMPILEFLAGS := -march=rv32imac -mabi=ilp32
 ARCH_OPTFLAGS := -O2
 
 LIBGCC := $(shell $(TOOLCHAIN_PREFIX)gcc $(GLOBAL_COMPILEFLAGS) $(ARCH_COMPILEFLAGS) $(GLOBAL_CFLAGS) -print-libgcc-file-name)

arch/riscv/thread.c

@@ -20,6 +20,7 @@
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
+#include <assert.h>
 #include <debug.h>
 #include <trace.h>
 #include <sys/types.h>
@@ -34,6 +35,8 @@ struct thread *_current_thread;
 
 static void initial_thread_func(void) __NO_RETURN;
 static void initial_thread_func(void) {
+    DEBUG_ASSERT(arch_ints_disabled());
+
     thread_t *ct = get_current_thread();
 
 #if LOCAL_TRACE
@@ -63,6 +66,8 @@ void arch_thread_initialize(thread_t *t) {
 }
 
 void arch_context_switch(thread_t *oldthread, thread_t *newthread) {
+    DEBUG_ASSERT(arch_ints_disabled());
+
     LTRACEF("old %p (%s), new %p (%s)\n", oldthread, oldthread->name, newthread, newthread->name);
 
     riscv32_context_switch(&oldthread->arch.cs_frame, &newthread->arch.cs_frame);

platform/sifive/include/platform/sifive.h

@@ -29,6 +29,14 @@
 
 #define CLINT_BASE 0x02000000
 #define PLIC_BASE  0x0c000000
+#define PRCI_BASE  0x10008000
 #define GPIO_BASE  0x10012000
 #define UART0_BASE 0x10013000
 
+#define GPIO_REG_VALUE      0
+#define GPIO_REG_INPUT_EN   1
+#define GPIO_REG_OUTPUT_EN  2
+#define GPIO_REG_PORT       3
+#define GPIO_REG_IOF_EN     14
+#define GPIO_REG_IOF_SEL    15
+

platform/sifive/platform.c

@@ -26,13 +26,15 @@
 #include <platform/interrupts.h>
 #include <platform/debug.h>
 #include <platform/timer.h>
+#include <platform/sifive.h>
 #include <sys/types.h>
 
 #include "platform_p.h"
 
 void platform_early_init(void) {
-    plic_early_init();
     sifive_uart_early_init();
+
+    plic_early_init();
 }
 
 void platform_init(void) {

platform/sifive/rules.mk

@@ -22,6 +22,6 @@ GLOBAL_DEFINES += PLATFORM_${VARIANT}=1
 # set some global defines based on capability
 GLOBAL_DEFINES += PLATFORM_HAS_DYNAMIC_TIMER=1
 GLOBAL_DEFINES += ARCH_RISCV_CLINT_BASE=0x02000000
-GLOBAL_DEFINES += ARCH_RISCV_MTIME_RATE=10000000 # 10Mhz
+GLOBAL_DEFINES += ARCH_RISCV_MTIME_RATE=32768
 
 include make/module.mk

platform/sifive/uart.c

@@ -34,7 +34,6 @@
 
 #define LOCAL_TRACE 0
 
-static volatile unsigned int *const gpio_base = (unsigned int *)GPIO_BASE;
 static volatile unsigned int *const uart_base = (unsigned int *)UART0_BASE;
 
 #define UART_TXDATA 0
@@ -50,6 +49,9 @@ static char uart_rx_buf_data[RXBUF_SIZE];
 static struct cbuf uart_rx_buf;
 
 void sifive_uart_write(int c) {
+    // wait for tx fifo to clear
+    while (uart_base[UART_TXDATA] & (1<<31))
+        ;
     uart_base[UART_TXDATA] = (c & 0xff);
 }
 
@@ -78,9 +80,7 @@ static enum handler_return sifive_uart_irq(void *unused) {
 }
 
 void sifive_uart_early_init(void) {
-    gpio_base[14] = (3<<16); // io function enable for pin 16/17
-
-    uart_base[UART_DIV] = 0x9be; // divisor
+    uart_base[UART_DIV] = SIFIVE_FREQ / 115200;
     uart_base[UART_TXCTRL] = 1; // txen
 }
 

scripts/do-sifive-e

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+
+
+export PROJECT=sifive-e-test
+TARGET=`pwd`/build-${PROJECT}/lk.elf
+
+make -j4
+
+SDK_DIR=${HOME}/src/freedom-e-sdk
+OPENOCD_DIR=${SDK_DIR}/riscv-openocd-0.10.0-2018.12.0-x86_64-linux-ubuntu14/bin
+
+${OPENOCD_DIR}/openocd -f ${SDK_DIR}/bsp/sifive-hifive1/openocd.cfg &
+
+${SDK_DIR}/riscv64-unknown-elf-gcc-8.1.0-2019.01.0-x86_64-linux-ubuntu14/bin/riscv64-unknown-elf-gdb \
+    ${TARGET} \
+    --batch -ex "set remotetimeout 240" \
+    -ex "target extended-remote localhost:3333" \
+    -ex "monitor reset halt" \
+    -ex "monitor flash protect 0 64 last off" \
+    -ex "load" \
+    -ex "monitor resume" \
+    -ex "monitor shutdown" \
+    -ex "quit"
+
+kill %1
+

target/sifive-e/rules.mk

@@ -1,11 +1,17 @@
 LOCAL_DIR := $(GET_LOCAL_DIR)
 
-GLOBAL_INCLUDES += $(LOCAL_DIR)/include
+MODULE := $(LOCAL_DIR)
 
 PLATFORM := sifive
 VARIANT := sifive_e
 
 MEMSIZE ?= 0x4000     # 16KB
+GLOBAL_DEFINES += TARGET_HAS_DEBUG_LED=1
 
-#include make/module.mk
+# target code will set the master frequency to 16Mhz
+GLOBAL_DEFINES += SIFIVE_FREQ=16000000
+
+MODULE_SRCS := $(LOCAL_DIR)/target.c
+
+include make/module.mk
 

target/sifive-e/target.c

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2019 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <target.h>
+#include <arch/arch_ops.h>
+#include <platform/sifive.h>
+
+static volatile unsigned int *const prci_base = (unsigned int *)PRCI_BASE;
+static volatile unsigned int *const gpio_base = (unsigned int *)GPIO_BASE;
+
+#define GPIO_LED_GREEN 19
+#define GPIO_LED_BLUE  21
+#define GPIO_LED_RED   22
+
+void target_early_init(void) {
+    // enable the external 16Mhz crystal
+    prci_base[1] = (1<<30); // hfxosc enable
+    while ((prci_base[1] & (1<<31)) == 0) // wait for hfxosc ready
+        ;
+
+    // program the pll bypass, we should be running at 16Mhz now
+    prci_base[2] = 0x00070df1;
+
+    // io function enable for pin 16/17, no IOF for all others
+    gpio_base[14] = (3<<16);
+
+    // turn our LED gpios off
+    gpio_base[GPIO_REG_PORT] |= (1u << GPIO_LED_GREEN) | (1u << GPIO_LED_BLUE) | (1u << GPIO_LED_RED);
+
+    // set the led gpios to output
+    gpio_base[GPIO_REG_OUTPUT_EN] |= (1u << GPIO_LED_GREEN) | (1u << GPIO_LED_BLUE) | (1u << GPIO_LED_RED);
+}
+
+void target_set_debug_led(unsigned int led, bool on) {
+    uint val = 0;
+    if (led == 0) {
+        val = 1u << GPIO_LED_GREEN;
+    } else if (led == 1) {
+        val = 1u << GPIO_LED_RED;
+    } else if (led == 2) {
+        val = 1u << GPIO_LED_BLUE;
+    }
+
+    // set and clear the LED gpios using atomic instructions
+    // polarity is inverted
+    if (on) {
+        __atomic_fetch_and((int *)&gpio_base[GPIO_REG_PORT], ~val, __ATOMIC_RELAXED);
+    } else {
+        __atomic_fetch_or((int *)&gpio_base[GPIO_REG_PORT], val, __ATOMIC_RELAXED);
+    }
+}
+
+void target_init(void) {
+}
+
+