lby/lk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[mp] restructure the sequence of how cpus are brought up

Browse Source 
- Move a bit of the shared logic of secondary bootstrapping into a new
  function, lk_secondary_cpu_entry_early() which sets the current cpu
  pointer before calling the first half of the secondary LK_INIT
  routines.
- Create the per cpu idle threads on the main cpu instead of the
  secondary as they come up.
- Tweak all of the SMP capable architectures to use this new path.
- Move the top level mp routines into a separate file top/mp.c
- A bit more correctly ifdef out more SMP code.
This commit is contained in:
Travis Geiselbrecht
2025-10-08 05:00:31 +00:00
parent e47183725d
commit e4d65228b5
15 changed files with 141 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
arch/arm/arm/arch.c
View File

@@ -164,20 +164,19 @@ void arm_secondary_entry(uint asm_cpu_num) {
sctlr |= (1<<12) | (1<<2); // enable i and dcache
arm_write_sctlr(sctlr);
/* run early secondary cpu init routines up to the threading level */
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
// Get us into thread context and run the initial secondary cpu init routines
lk_secondary_cpu_entry_early();
arch_mp_init_percpu();
LTRACEF("cpu num %d\n", cpu);
LTRACEF("sctlr 0x%x\n", arm_read_sctlr());
LTRACEF("actlr 0x%x\n", arm_read_actlr());
dprintf(INFO, "ARM: secondary cpu %u started\n", arch_curr_cpu_num());
/* we're done, tell the main cpu we're up */
atomic_add(&secondaries_to_init, -1);
smp_mb();
__asm__ volatile("sev");
// Finish secondary cpu initialization and enter the scheduler
lk_secondary_cpu_entry();
}
#endif

4
arch/arm64/arch.c
View File

@@ -55,6 +55,10 @@ void arm64_early_init_percpu(void) {
ARM64_WRITE_SYSREG(MDSCR_EL1, 0UL); // disable debug
// clear the tpidr registers
ARM64_WRITE_SYSREG(TPIDR_EL0, 0UL);
ARM64_WRITE_SYSREG(TPIDRRO_EL0, 0UL);
// TODO: read feature bits on cpu 0
// TODO: enable cycle counter if present

13
arch/arm64/fpu.c
View File

@@ -27,8 +27,7 @@ static void arm64_fpu_load_state(struct thread *t) {
fpstate->current_cpu = cpu;
current_fpstate[cpu] = fpstate;
STATIC_ASSERT(sizeof(fpstate->regs) == 16 * 32);
STATIC_ASSERT(sizeof(fpstate->regs) == (size_t)16 * 32);
__asm__ volatile(
".arch_extension fp\n"
"ldp q0, q1, [%0, #(0 * 32)]\n"
@@ -49,8 +48,8 @@ static void arm64_fpu_load_state(struct thread *t) {
"ldp q30, q31, [%0, #(15 * 32)]\n"
"msr fpcr, %1\n"
"msr fpsr, %2\n"
".arch_extension nofp\n"
:: "r"(fpstate), "r"((uint64_t)fpstate->fpcr), "r"((uint64_t)fpstate->fpsr));
".arch_extension nofp\n" ::"r"(fpstate),
"r"((uint64_t)fpstate->fpcr), "r"((uint64_t)fpstate->fpsr));
}
void arm64_fpu_save_state(struct thread *t) {
@@ -86,13 +85,13 @@ void arm64_fpu_save_state(struct thread *t) {
}
void arm64_fpu_exception(struct arm64_iframe_long *iframe) {
uint32_t cpacr = ARM64_READ_SYSREG(cpacr_el1);
uint64_t cpacr = ARM64_READ_SYSREG(cpacr_el1);
if (((cpacr >> 20) & 3) != 3) {
cpacr |= 3 << 20;
ARM64_WRITE_SYSREG(cpacr_el1, cpacr);
thread_t *t = get_current_thread();
if (likely(t))
if (likely(t)) {
arm64_fpu_load_state(t);
return;
}
}
}

8
arch/arm64/mp.c
View File

@@ -16,6 +16,7 @@
#include <lk/main.h>
#include <lk/trace.h>
#include <platform/interrupts.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
@@ -130,13 +131,14 @@ void arm64_secondary_entry(ulong asm_cpu_num) {
arm64_early_init_percpu();
/* run early secondary cpu init routines up to the threading level */
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
// Get us into thread context and run the initial secondary cpu init routines
lk_secondary_cpu_entry_early();
arm64_mp_init_percpu();
LTRACEF("cpu num %d\n", cpu);
dprintf(INFO, "ARM64: secondary cpu %u started, mpidr %#" PRIx64 "\n", arch_curr_cpu_num(), ARM64_READ_SYSREG(mpidr_el1));
// Finish secondary cpu initialization and enter the scheduler
lk_secondary_cpu_entry();
}
#endif // WITH_SMP

1
arch/riscv/arch.c
View File

@@ -84,7 +84,6 @@ void arch_early_init(void) {
// later init per cpu
void riscv_init_percpu(void) {
dprintf(INFO, "RISCV: percpu cpu num %#x hart id %#x\n", arch_curr_cpu_num(), riscv_current_hart());
#if WITH_SMP
// enable software interrupts, used for inter-processor-interrupts
riscv_csr_set(RISCV_CSR_XIE, RISCV_CSR_XIE_SIE);

8
arch/riscv/mp.c
View File

@@ -133,16 +133,18 @@ void riscv_secondary_entry(uint hart_id, uint __unused, uint cpu_id) {
riscv_mmu_init_secondaries();
#endif
// run early secondary cpu init routines up to the threading level
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
// Get us into thread context and run the initial secondary cpu init routines
lk_secondary_cpu_entry_early();
// run threading level initialization on this cpu
riscv_init_percpu();
dprintf(INFO, "RISCV: secondary hart coming up: mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n",
dprintf(INFO, "RISCV: secondary hart started: cpu %u, mvendorid %#lx marchid %#lx mimpid %#lx mhartid %#x\n",
arch_curr_cpu_num(),
riscv_get_mvendorid(), riscv_get_marchid(),
riscv_get_mimpid(), riscv_current_hart());
// Finish secondary cpu initialization and enter the scheduler
lk_secondary_cpu_entry();
}

7
arch/x86/mp.c
View File

@@ -96,11 +96,12 @@ void x86_secondary_entry(uint cpu_num) {
x86_early_init_percpu();
// run early secondary cpu init routines up to the threading level
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
// Get us into thread context and run the initial secondary cpu init routines
lk_secondary_cpu_entry_early();
dprintf(INFO, "SMP: secondary cpu %u started, apic id %u\n", arch_curr_cpu_num(), apic_id);
dprintf(INFO, "X86: secondary cpu %u started, apic id %u\n", arch_curr_cpu_num(), apic_id);
// Finish secondary cpu initialization and enter the scheduler
lk_secondary_cpu_entry();
// should never get here except for an error condition

2
dev/timer/arm_generic/arm_generic_timer.c
View File

@@ -327,7 +327,7 @@ static void arm_generic_timer_init_secondary_cpu(uint level) {
/* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */
LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu,
arm_generic_timer_init_secondary_cpu,
LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS);
LK_INIT_LEVEL_PLATFORM_EARLY, LK_INIT_FLAG_SECONDARY_CPUS);
static void arm_generic_timer_resume_cpu(uint level) {
/* Always trigger a timer interrupt on each cpu for now */

3
kernel/include/kernel/thread.h
View File

@@ -158,8 +158,11 @@ typedef struct thread {
void thread_init_early(void);
void thread_init(void);
void thread_become_idle(void) __NO_RETURN;
#if WITH_SMP
void thread_secondary_cpu_init_early(void);
void thread_secondary_cpu_entry(void) __NO_RETURN;
void thread_create_secondary_cpu_idle_thread(uint cpu);
#endif
void thread_set_name(const char *name);
void thread_set_priority(int priority);
thread_t *thread_create(const char *name, thread_start_routine entry, void *arg, int priority, size_t stack_size);

33
kernel/thread.c
View File

@@ -859,7 +859,7 @@ void thread_become_idle(void) {
#if WITH_SMP
char name[16];
snprintf(name, sizeof(name), "idle %d", arch_curr_cpu_num());
snprintf(name, sizeof(name), "idle %u", arch_curr_cpu_num());
thread_set_name(name);
#else
thread_set_name("idle");
@@ -880,21 +880,27 @@ void thread_become_idle(void) {
idle_thread_routine();
}
/* create an idle thread for the cpu we're on, and start scheduling */
#if WITH_SMP
// Get this secondary cpu onto thread context
void thread_secondary_cpu_init_early(void) {
DEBUG_ASSERT(arch_ints_disabled());
/* construct an idle thread to cover our cpu */
// Look up our idle thread and mark it as current on this cpu
uint cpu = arch_curr_cpu_num();
thread_t *t = idle_thread(cpu);
DEBUG_ASSERT(t && t->state == THREAD_RUNNING && t->flags & THREAD_FLAG_IDLE);
set_current_thread(t);
}
// Construct a secondary's cpu idle thread such that it appears to already be running
void thread_create_secondary_cpu_idle_thread(uint cpu) {
thread_t *t = idle_thread(cpu);
char name[16];
snprintf(name, sizeof(name), "idle %u", cpu);
init_thread_struct(t, name);
thread_set_pinned_cpu(t, cpu);
/* half construct this thread, since we're already running */
// Half construct this thread as if it were running at max priority, since the secondary cpu will
// start executing this thread when it starts.
t->priority = HIGHEST_PRIORITY;
t->state = THREAD_RUNNING;
t->flags = THREAD_FLAG_DETACHED | THREAD_FLAG_IDLE;
@@ -903,27 +909,32 @@ void thread_secondary_cpu_init_early(void) {
wait_queue_init(&t->retcode_wait_queue);
THREAD_LOCK(state);
list_add_head(&thread_list, &t->thread_list_node);
set_current_thread(t);
THREAD_UNLOCK(state);
}
// We should be on the idle thread for the secondary cpu, drop to idle priority and
// start properly scheduling by enabling interrupts and yielding to the scheduler.
void thread_secondary_cpu_entry(void) {
uint cpu = arch_curr_cpu_num();
thread_t *t = get_current_thread();
DEBUG_ASSERT(t && t->state == THREAD_RUNNING && t->flags & THREAD_FLAG_IDLE);
t->priority = IDLE_PRIORITY;
// Mark the local cpu as active and idle
mp_set_curr_cpu_active(true);
mp_set_cpu_idle(cpu);
/* enable interrupts and start the scheduler on this cpu */
// Enable interrupts and start the scheduler on this cpu
arch_enable_ints();
thread_yield();
// Fall through to the idle thread routine
idle_thread_routine();
}
#endif // WITH_SMP
static const char *thread_state_to_str(enum thread_state state) {
switch (state) {

3
lib/fdtwalk/helpers.cpp
View File

@@ -14,6 +14,7 @@
#include <lk/err.h>
#include <lk/main.h>
#include <lk/trace.h>
#include <kernel/thread.h>
#include <stdio.h>
#include <sys/types.h>
#if WITH_KERNEL_VM
@@ -239,7 +240,7 @@ status_t fdtwalk_setup_cpus_arm(const void *fdt) {
/* boot the secondary cpus using the Power State Coordintion Interface */
for (size_t i = 1; i < cpu_count; i++) {
/* note: assumes cpuids are numbered like MPIDR 0:0:0:N */
dprintf(INFO, "ARM: starting cpu %#x\n", cpus[i].id);
dprintf(INFO, "ARM: starting cpu %u\n", cpus[i].id);
int ret = psci_cpu_on(cpus[i].id, phys_entry, i);
if (ret != 0) {
printf("ERROR: psci CPU_ON returns %d\n", ret);

17
top/include/lk/main.h
View File

@@ -15,8 +15,23 @@ __BEGIN_CDECLS
// Possible boot args passed from various arch's start.S
extern ulong lk_boot_args[4];
// Main entry point to the OS on the boot cpu. Called from low level arch code after getting the
// boot cpu into enough of a known state to enter C code.
void lk_main(ulong arg0, ulong arg1, ulong arg2, ulong arg3) __NO_RETURN __EXTERNALLY_VISIBLE;
void lk_secondary_cpu_entry(void);
#if WITH_SMP
// Before starting any secondary cpus, the boot cpu should call this function to set up
// the secondary cpu idle and bootstrap threads.
void lk_init_secondary_cpus(uint secondary_cpu_count);
// High level entry point for secondary cpus. Called from low level arch code to bootstrap the
// threading system on the secondary cpu and call any secondary cpu init routines up to
// LK_INIT_LEVEL_THREADING.
void lk_secondary_cpu_entry_early(void);
// Final entry point for secondary cpus from arch code, running the rest of the secondary cpu
// init routines and entering the scheduler. Does not return.
void lk_secondary_cpu_entry(void) __NO_RETURN;
#endif
__END_CDECLS

50
top/main.c
View File

@@ -33,11 +33,6 @@ extern void (*__ctor_end[])(void);
extern int __bss_start;
extern int _end;
#if WITH_SMP
static thread_t *secondary_bootstrap_threads[SMP_MAX_CPUS - 1];
static uint secondary_bootstrap_thread_count;
#endif
static int bootstrap2(void *arg);
static void call_constructors(void) {
@@ -54,7 +49,7 @@ static void call_constructors(void) {
}
}
/* called from arch code */
// Main C entry point of the system, called from arch code on the boot cpu.
void lk_main(ulong arg0, ulong arg1, ulong arg2, ulong arg3) {
// save the boot args
lk_boot_args[0] = arg0;
@@ -135,46 +130,3 @@ static int bootstrap2(void *arg) {
return 0;
}
#if WITH_SMP
void lk_secondary_cpu_entry(void) {
uint cpu = arch_curr_cpu_num();
if (cpu > secondary_bootstrap_thread_count) {
dprintf(CRITICAL, "Invalid secondary cpu num %d, SMP_MAX_CPUS %d, secondary_bootstrap_thread_count %d\n",
cpu, SMP_MAX_CPUS, secondary_bootstrap_thread_count);
return;
}
thread_secondary_cpu_init_early();
thread_resume(secondary_bootstrap_threads[cpu - 1]);
dprintf(SPEW, "entering scheduler on cpu %d\n", cpu);
thread_secondary_cpu_entry();
}
static int secondary_cpu_bootstrap2(void *arg) {
/* secondary cpu initialize from threading level up. 0 to threading was handled in arch */
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_THREADING, LK_INIT_LEVEL_LAST);
return 0;
}
void lk_init_secondary_cpus(uint secondary_cpu_count) {
if (secondary_cpu_count >= SMP_MAX_CPUS) {
dprintf(CRITICAL, "Invalid secondary_cpu_count %d, SMP_MAX_CPUS %d\n",
secondary_cpu_count, SMP_MAX_CPUS);
secondary_cpu_count = SMP_MAX_CPUS - 1;
}
for (uint i = 0; i < secondary_cpu_count; i++) {
dprintf(SPEW, "creating bootstrap completion thread for cpu %d\n", i + 1);
thread_t *t = thread_create("secondarybootstrap2",
&secondary_cpu_bootstrap2, NULL,
DEFAULT_PRIORITY, DEFAULT_STACK_SIZE);
t->pinned_cpu = i + 1;
thread_detach(t);
secondary_bootstrap_threads[i] = t;
}
secondary_bootstrap_thread_count = secondary_cpu_count;
}
#endif

66
top/mp.c Normal file
View File

@@ -0,0 +1,66 @@
// Copyright (c) 2013-2015 Travis Geiselbrecht
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
#include <assert.h>
#include <kernel/thread.h>
#include <lk/compiler.h>
#include <lk/debug.h>
#include <lk/init.h>
#include <lk/main.h>
// Top level entry points for secondary cpus
#if WITH_SMP
static thread_t *secondary_bootstrap_threads[SMP_MAX_CPUS - 1];
static uint secondary_bootstrap_thread_count;
void lk_secondary_cpu_entry_early(void) {
// get the cpu into threading context
thread_secondary_cpu_init_early();
// run early secondary cpu init routines up to the threading level
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
}
void lk_secondary_cpu_entry(void) {
uint cpu = arch_curr_cpu_num();
DEBUG_ASSERT(cpu <= secondary_bootstrap_thread_count);
thread_resume(secondary_bootstrap_threads[cpu - 1]);
dprintf(SPEW, "entering scheduler on cpu %d\n", cpu);
thread_secondary_cpu_entry();
}
// Secondary cpu bootstrap thread, which gives enough thread context to run the secondary cpu init routines
// from LK_INIT_LEVEL_THREADING to LK_INIT_LEVEL_LAST.
static int secondary_cpu_bootstrap_thread(void *arg) {
// Secondary cpu initialize from threading level up. 0 to threading was handled in arch
lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_THREADING, LK_INIT_LEVEL_LAST);
return 0;
}
void lk_init_secondary_cpus(uint secondary_cpu_count) {
if (secondary_cpu_count >= SMP_MAX_CPUS) {
dprintf(CRITICAL, "Invalid secondary_cpu_count %d, SMP_MAX_CPUS %d\n",
secondary_cpu_count, SMP_MAX_CPUS);
secondary_cpu_count = SMP_MAX_CPUS - 1;
}
// Construct the idle and bootstrap threads for each secondary cpu
for (uint i = 0; i < secondary_cpu_count; i++) {
thread_create_secondary_cpu_idle_thread(i + 1);
dprintf(SPEW, "creating bootstrap completion thread for cpu %d\n", i + 1);
thread_t *t = thread_create("secondarybootstrap2",
&secondary_cpu_bootstrap_thread, NULL,
DEFAULT_PRIORITY, DEFAULT_STACK_SIZE);
thread_set_pinned_cpu(t, i + 1);
secondary_bootstrap_threads[i] = t;
thread_detach(t);
}
secondary_bootstrap_thread_count = secondary_cpu_count;
}
#endif // WITH_SMP

2
top/rules.mk
View File

@@ -14,6 +14,8 @@ MODULE_SRCS := \
$(LOCAL_DIR)/debug.c \
$(LOCAL_DIR)/init.c \
$(LOCAL_DIR)/main.c \
$(LOCAL_DIR)/mp.c \
MODULE_OPTIONS := extra_warnings