diff --git a/arch/arm/arm/arch.c b/arch/arm/arm/arch.c index ae9fb49d..fdcb6937 100644 --- a/arch/arm/arm/arch.c +++ b/arch/arm/arm/arch.c @@ -164,20 +164,19 @@ void arm_secondary_entry(uint asm_cpu_num) { sctlr |= (1<<12) | (1<<2); // enable i and dcache arm_write_sctlr(sctlr); - /* run early secondary cpu init routines up to the threading level */ - lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); + // Get us into thread context and run the initial secondary cpu init routines + lk_secondary_cpu_entry_early(); arch_mp_init_percpu(); - LTRACEF("cpu num %d\n", cpu); - LTRACEF("sctlr 0x%x\n", arm_read_sctlr()); - LTRACEF("actlr 0x%x\n", arm_read_actlr()); + dprintf(INFO, "ARM: secondary cpu %u started\n", arch_curr_cpu_num()); /* we're done, tell the main cpu we're up */ atomic_add(&secondaries_to_init, -1); smp_mb(); __asm__ volatile("sev"); + // Finish secondary cpu initialization and enter the scheduler lk_secondary_cpu_entry(); } #endif diff --git a/arch/arm64/arch.c b/arch/arm64/arch.c index f4578e9d..6af6d993 100644 --- a/arch/arm64/arch.c +++ b/arch/arm64/arch.c @@ -55,6 +55,10 @@ void arm64_early_init_percpu(void) { ARM64_WRITE_SYSREG(MDSCR_EL1, 0UL); // disable debug + // clear the tpidr registers + ARM64_WRITE_SYSREG(TPIDR_EL0, 0UL); + ARM64_WRITE_SYSREG(TPIDRRO_EL0, 0UL); + // TODO: read feature bits on cpu 0 // TODO: enable cycle counter if present diff --git a/arch/arm64/fpu.c b/arch/arm64/fpu.c index 374a1dd5..631c10fa 100644 --- a/arch/arm64/fpu.c +++ b/arch/arm64/fpu.c @@ -27,8 +27,7 @@ static void arm64_fpu_load_state(struct thread *t) { fpstate->current_cpu = cpu; current_fpstate[cpu] = fpstate; - - STATIC_ASSERT(sizeof(fpstate->regs) == 16 * 32); + STATIC_ASSERT(sizeof(fpstate->regs) == (size_t)16 * 32); __asm__ volatile( ".arch_extension fp\n" "ldp q0, q1, [%0, #(0 * 32)]\n" @@ -49,8 +48,8 @@ static void arm64_fpu_load_state(struct thread *t) { "ldp q30, q31, [%0, #(15 * 32)]\n" "msr fpcr, %1\n" "msr fpsr, %2\n" - ".arch_extension nofp\n" - :: "r"(fpstate), "r"((uint64_t)fpstate->fpcr), "r"((uint64_t)fpstate->fpsr)); + ".arch_extension nofp\n" ::"r"(fpstate), + "r"((uint64_t)fpstate->fpcr), "r"((uint64_t)fpstate->fpsr)); } void arm64_fpu_save_state(struct thread *t) { @@ -86,13 +85,13 @@ void arm64_fpu_save_state(struct thread *t) { } void arm64_fpu_exception(struct arm64_iframe_long *iframe) { - uint32_t cpacr = ARM64_READ_SYSREG(cpacr_el1); + uint64_t cpacr = ARM64_READ_SYSREG(cpacr_el1); if (((cpacr >> 20) & 3) != 3) { cpacr |= 3 << 20; ARM64_WRITE_SYSREG(cpacr_el1, cpacr); thread_t *t = get_current_thread(); - if (likely(t)) + if (likely(t)) { arm64_fpu_load_state(t); - return; + } } } diff --git a/arch/arm64/mp.c b/arch/arm64/mp.c index 651edd29..70dc6f69 100644 --- a/arch/arm64/mp.c +++ b/arch/arm64/mp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -130,13 +131,14 @@ void arm64_secondary_entry(ulong asm_cpu_num) { arm64_early_init_percpu(); - /* run early secondary cpu init routines up to the threading level */ - lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); + // Get us into thread context and run the initial secondary cpu init routines + lk_secondary_cpu_entry_early(); arm64_mp_init_percpu(); - LTRACEF("cpu num %d\n", cpu); + dprintf(INFO, "ARM64: secondary cpu %u started, mpidr %#" PRIx64 "\n", arch_curr_cpu_num(), ARM64_READ_SYSREG(mpidr_el1)); + // Finish secondary cpu initialization and enter the scheduler lk_secondary_cpu_entry(); } #endif // WITH_SMP diff --git a/arch/riscv/arch.c b/arch/riscv/arch.c index e1a2bcc1..41b8cec1 100644 --- a/arch/riscv/arch.c +++ b/arch/riscv/arch.c @@ -84,7 +84,6 @@ void arch_early_init(void) { // later init per cpu void riscv_init_percpu(void) { - dprintf(INFO, "RISCV: percpu cpu num %#x hart id %#x\n", arch_curr_cpu_num(), riscv_current_hart()); #if WITH_SMP // enable software interrupts, used for inter-processor-interrupts riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_SIE); diff --git a/arch/riscv/mp.c b/arch/riscv/mp.c index 2fcf72b9..6d984663 100644 --- a/arch/riscv/mp.c +++ b/arch/riscv/mp.c @@ -133,16 +133,18 @@ void riscv_secondary_entry(uint hart_id, uint __unused, uint cpu_id) { riscv_mmu_init_secondaries(); #endif - // run early secondary cpu init routines up to the threading level - lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); + // Get us into thread context and run the initial secondary cpu init routines + lk_secondary_cpu_entry_early(); // run threading level initialization on this cpu riscv_init_percpu(); - dprintf(INFO, "RISCV: secondary hart coming up: mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n", + dprintf(INFO, "RISCV: secondary hart started: cpu %u, mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n", + arch_curr_cpu_num(), riscv_get_mvendorid(), riscv_get_marchid(), riscv_get_mimpid(), riscv_current_hart()); + // Finish secondary cpu initialization and enter the scheduler lk_secondary_cpu_entry(); } diff --git a/arch/x86/mp.c b/arch/x86/mp.c index d0345a74..c1099ac7 100644 --- a/arch/x86/mp.c +++ b/arch/x86/mp.c @@ -96,11 +96,12 @@ void x86_secondary_entry(uint cpu_num) { x86_early_init_percpu(); - // run early secondary cpu init routines up to the threading level - lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); + // Get us into thread context and run the initial secondary cpu init routines + lk_secondary_cpu_entry_early(); - dprintf(INFO, "SMP: secondary cpu %u started, apic id %u\n", arch_curr_cpu_num(), apic_id); + dprintf(INFO, "X86: secondary cpu %u started, apic id %u\n", arch_curr_cpu_num(), apic_id); + // Finish secondary cpu initialization and enter the scheduler lk_secondary_cpu_entry(); // should never get here except for an error condition diff --git a/dev/timer/arm_generic/arm_generic_timer.c b/dev/timer/arm_generic/arm_generic_timer.c index e9781e3d..4f78e2ef 100644 --- a/dev/timer/arm_generic/arm_generic_timer.c +++ b/dev/timer/arm_generic/arm_generic_timer.c @@ -327,7 +327,7 @@ static void arm_generic_timer_init_secondary_cpu(uint level) { /* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */ LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu, arm_generic_timer_init_secondary_cpu, - LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS); + LK_INIT_LEVEL_PLATFORM_EARLY, LK_INIT_FLAG_SECONDARY_CPUS); static void arm_generic_timer_resume_cpu(uint level) { /* Always trigger a timer interrupt on each cpu for now */ diff --git a/kernel/include/kernel/thread.h b/kernel/include/kernel/thread.h index ff9867b8..c633bae0 100644 --- a/kernel/include/kernel/thread.h +++ b/kernel/include/kernel/thread.h @@ -158,8 +158,11 @@ typedef struct thread { void thread_init_early(void); void thread_init(void); void thread_become_idle(void) __NO_RETURN; +#if WITH_SMP void thread_secondary_cpu_init_early(void); void thread_secondary_cpu_entry(void) __NO_RETURN; +void thread_create_secondary_cpu_idle_thread(uint cpu); +#endif void thread_set_name(const char *name); void thread_set_priority(int priority); thread_t *thread_create(const char *name, thread_start_routine entry, void *arg, int priority, size_t stack_size); diff --git a/kernel/thread.c b/kernel/thread.c index 80c55432..fadeb924 100644 --- a/kernel/thread.c +++ b/kernel/thread.c @@ -859,7 +859,7 @@ void thread_become_idle(void) { #if WITH_SMP char name[16]; - snprintf(name, sizeof(name), "idle %d", arch_curr_cpu_num()); + snprintf(name, sizeof(name), "idle %u", arch_curr_cpu_num()); thread_set_name(name); #else thread_set_name("idle"); @@ -880,21 +880,27 @@ void thread_become_idle(void) { idle_thread_routine(); } -/* create an idle thread for the cpu we're on, and start scheduling */ - +#if WITH_SMP +// Get this secondary cpu onto thread context void thread_secondary_cpu_init_early(void) { DEBUG_ASSERT(arch_ints_disabled()); - - /* construct an idle thread to cover our cpu */ + // Look up our idle thread and mark it as current on this cpu uint cpu = arch_curr_cpu_num(); thread_t *t = idle_thread(cpu); + DEBUG_ASSERT(t && t->state == THREAD_RUNNING && t->flags & THREAD_FLAG_IDLE); + set_current_thread(t); +} + +// Construct a secondary's cpu idle thread such that it appears to already be running +void thread_create_secondary_cpu_idle_thread(uint cpu) { + thread_t *t = idle_thread(cpu); char name[16]; snprintf(name, sizeof(name), "idle %u", cpu); init_thread_struct(t, name); - thread_set_pinned_cpu(t, cpu); - /* half construct this thread, since we're already running */ + // Half construct this thread as if it were running at max priority, since the secondary cpu will + // start executing this thread when it starts. t->priority = HIGHEST_PRIORITY; t->state = THREAD_RUNNING; t->flags = THREAD_FLAG_DETACHED | THREAD_FLAG_IDLE; @@ -903,27 +909,32 @@ void thread_secondary_cpu_init_early(void) { wait_queue_init(&t->retcode_wait_queue); THREAD_LOCK(state); - list_add_head(&thread_list, &t->thread_list_node); - set_current_thread(t); - THREAD_UNLOCK(state); } +// We should be on the idle thread for the secondary cpu, drop to idle priority and +// start properly scheduling by enabling interrupts and yielding to the scheduler. void thread_secondary_cpu_entry(void) { uint cpu = arch_curr_cpu_num(); thread_t *t = get_current_thread(); + + DEBUG_ASSERT(t && t->state == THREAD_RUNNING && t->flags & THREAD_FLAG_IDLE); + t->priority = IDLE_PRIORITY; + // Mark the local cpu as active and idle mp_set_curr_cpu_active(true); mp_set_cpu_idle(cpu); - /* enable interrupts and start the scheduler on this cpu */ + // Enable interrupts and start the scheduler on this cpu arch_enable_ints(); thread_yield(); + // Fall through to the idle thread routine idle_thread_routine(); } +#endif // WITH_SMP static const char *thread_state_to_str(enum thread_state state) { switch (state) { diff --git a/lib/fdtwalk/helpers.cpp b/lib/fdtwalk/helpers.cpp index 2e46d071..f096e5ac 100644 --- a/lib/fdtwalk/helpers.cpp +++ b/lib/fdtwalk/helpers.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #if WITH_KERNEL_VM @@ -239,7 +240,7 @@ status_t fdtwalk_setup_cpus_arm(const void *fdt) { /* boot the secondary cpus using the Power State Coordintion Interface */ for (size_t i = 1; i < cpu_count; i++) { /* note: assumes cpuids are numbered like MPIDR 0:0:0:N */ - dprintf(INFO, "ARM: starting cpu %#x\n", cpus[i].id); + dprintf(INFO, "ARM: starting cpu %u\n", cpus[i].id); int ret = psci_cpu_on(cpus[i].id, phys_entry, i); if (ret != 0) { printf("ERROR: psci CPU_ON returns %d\n", ret); diff --git a/top/include/lk/main.h b/top/include/lk/main.h index 6bfc230b..5beae97e 100644 --- a/top/include/lk/main.h +++ b/top/include/lk/main.h @@ -15,8 +15,23 @@ __BEGIN_CDECLS // Possible boot args passed from various arch's start.S extern ulong lk_boot_args[4]; +// Main entry point to the OS on the boot cpu. Called from low level arch code after getting the +// boot cpu into enough of a known state to enter C code. void lk_main(ulong arg0, ulong arg1, ulong arg2, ulong arg3) __NO_RETURN __EXTERNALLY_VISIBLE; -void lk_secondary_cpu_entry(void); + +#if WITH_SMP +// Before starting any secondary cpus, the boot cpu should call this function to set up +// the secondary cpu idle and bootstrap threads. void lk_init_secondary_cpus(uint secondary_cpu_count); +// High level entry point for secondary cpus. Called from low level arch code to bootstrap the +// threading system on the secondary cpu and call any secondary cpu init routines up to +// LK_INIT_LEVEL_THREADING. +void lk_secondary_cpu_entry_early(void); + +// Final entry point for secondary cpus from arch code, running the rest of the secondary cpu +// init routines and entering the scheduler. Does not return. +void lk_secondary_cpu_entry(void) __NO_RETURN; +#endif + __END_CDECLS diff --git a/top/main.c b/top/main.c index efba5932..6447ae8b 100644 --- a/top/main.c +++ b/top/main.c @@ -33,11 +33,6 @@ extern void (*__ctor_end[])(void); extern int __bss_start; extern int _end; -#if WITH_SMP -static thread_t *secondary_bootstrap_threads[SMP_MAX_CPUS - 1]; -static uint secondary_bootstrap_thread_count; -#endif - static int bootstrap2(void *arg); static void call_constructors(void) { @@ -54,7 +49,7 @@ static void call_constructors(void) { } } -/* called from arch code */ +// Main C entry point of the system, called from arch code on the boot cpu. void lk_main(ulong arg0, ulong arg1, ulong arg2, ulong arg3) { // save the boot args lk_boot_args[0] = arg0; @@ -135,46 +130,3 @@ static int bootstrap2(void *arg) { return 0; } - -#if WITH_SMP -void lk_secondary_cpu_entry(void) { - uint cpu = arch_curr_cpu_num(); - - if (cpu > secondary_bootstrap_thread_count) { - dprintf(CRITICAL, "Invalid secondary cpu num %d, SMP_MAX_CPUS %d, secondary_bootstrap_thread_count %d\n", - cpu, SMP_MAX_CPUS, secondary_bootstrap_thread_count); - return; - } - - thread_secondary_cpu_init_early(); - thread_resume(secondary_bootstrap_threads[cpu - 1]); - - dprintf(SPEW, "entering scheduler on cpu %d\n", cpu); - thread_secondary_cpu_entry(); -} - -static int secondary_cpu_bootstrap2(void *arg) { - /* secondary cpu initialize from threading level up. 0 to threading was handled in arch */ - lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_THREADING, LK_INIT_LEVEL_LAST); - - return 0; -} - -void lk_init_secondary_cpus(uint secondary_cpu_count) { - if (secondary_cpu_count >= SMP_MAX_CPUS) { - dprintf(CRITICAL, "Invalid secondary_cpu_count %d, SMP_MAX_CPUS %d\n", - secondary_cpu_count, SMP_MAX_CPUS); - secondary_cpu_count = SMP_MAX_CPUS - 1; - } - for (uint i = 0; i < secondary_cpu_count; i++) { - dprintf(SPEW, "creating bootstrap completion thread for cpu %d\n", i + 1); - thread_t *t = thread_create("secondarybootstrap2", - &secondary_cpu_bootstrap2, NULL, - DEFAULT_PRIORITY, DEFAULT_STACK_SIZE); - t->pinned_cpu = i + 1; - thread_detach(t); - secondary_bootstrap_threads[i] = t; - } - secondary_bootstrap_thread_count = secondary_cpu_count; -} -#endif diff --git a/top/mp.c b/top/mp.c new file mode 100644 index 00000000..4f7abc7f --- /dev/null +++ b/top/mp.c @@ -0,0 +1,66 @@ +// Copyright (c) 2013-2015 Travis Geiselbrecht +// +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT +#include +#include +#include +#include +#include +#include + +// Top level entry points for secondary cpus + +#if WITH_SMP +static thread_t *secondary_bootstrap_threads[SMP_MAX_CPUS - 1]; +static uint secondary_bootstrap_thread_count; + +void lk_secondary_cpu_entry_early(void) { + // get the cpu into threading context + thread_secondary_cpu_init_early(); + + // run early secondary cpu init routines up to the threading level + lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1); +} + +void lk_secondary_cpu_entry(void) { + uint cpu = arch_curr_cpu_num(); + DEBUG_ASSERT(cpu <= secondary_bootstrap_thread_count); + + thread_resume(secondary_bootstrap_threads[cpu - 1]); + + dprintf(SPEW, "entering scheduler on cpu %d\n", cpu); + thread_secondary_cpu_entry(); +} + +// Secondary cpu bootstrap thread, which gives enough thread context to run the secondary cpu init routines +// from LK_INIT_LEVEL_THREADING to LK_INIT_LEVEL_LAST. +static int secondary_cpu_bootstrap_thread(void *arg) { + // Secondary cpu initialize from threading level up. 0 to threading was handled in arch + lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_THREADING, LK_INIT_LEVEL_LAST); + + return 0; +} + +void lk_init_secondary_cpus(uint secondary_cpu_count) { + if (secondary_cpu_count >= SMP_MAX_CPUS) { + dprintf(CRITICAL, "Invalid secondary_cpu_count %d, SMP_MAX_CPUS %d\n", + secondary_cpu_count, SMP_MAX_CPUS); + secondary_cpu_count = SMP_MAX_CPUS - 1; + } + // Construct the idle and bootstrap threads for each secondary cpu + for (uint i = 0; i < secondary_cpu_count; i++) { + thread_create_secondary_cpu_idle_thread(i + 1); + + dprintf(SPEW, "creating bootstrap completion thread for cpu %d\n", i + 1); + thread_t *t = thread_create("secondarybootstrap2", + &secondary_cpu_bootstrap_thread, NULL, + DEFAULT_PRIORITY, DEFAULT_STACK_SIZE); + thread_set_pinned_cpu(t, i + 1); + secondary_bootstrap_threads[i] = t; + thread_detach(t); + } + secondary_bootstrap_thread_count = secondary_cpu_count; +} +#endif // WITH_SMP diff --git a/top/rules.mk b/top/rules.mk index c1b4789b..956b18d2 100644 --- a/top/rules.mk +++ b/top/rules.mk @@ -14,6 +14,8 @@ MODULE_SRCS := \ $(LOCAL_DIR)/debug.c \ $(LOCAL_DIR)/init.c \ $(LOCAL_DIR)/main.c \ + $(LOCAL_DIR)/mp.c \ + MODULE_OPTIONS := extra_warnings