[mp] restructure the sequence of how cpus are brought up

- Move a bit of the shared logic of secondary bootstrapping into a new
  function, lk_secondary_cpu_entry_early() which sets the current cpu
  pointer before calling the first half of the secondary LK_INIT
  routines.
- Create the per cpu idle threads on the main cpu instead of the
  secondary as they come up.
- Tweak all of the SMP capable architectures to use this new path.
- Move the top level mp routines into a separate file top/mp.c
- A bit more correctly ifdef out more SMP code.

top/include/lk/main.h

@@ -15,8 +15,23 @@ __BEGIN_CDECLS
 // Possible boot args passed from various arch's start.S
 extern ulong lk_boot_args[4];
 
+// Main entry point to the OS on the boot cpu. Called from low level arch code after getting the
+// boot cpu into enough of a known state to enter C code.
 void lk_main(ulong arg0, ulong arg1, ulong arg2, ulong arg3) __NO_RETURN __EXTERNALLY_VISIBLE;
-void lk_secondary_cpu_entry(void);
+
+#if WITH_SMP
+// Before starting any secondary cpus, the boot cpu should call this function to set up
+// the secondary cpu idle and bootstrap threads.
 void lk_init_secondary_cpus(uint secondary_cpu_count);
 
+// High level entry point for secondary cpus. Called from low level arch code to bootstrap the
+// threading system on the secondary cpu and call any secondary cpu init routines up to
+// LK_INIT_LEVEL_THREADING.
+void lk_secondary_cpu_entry_early(void);
+
+// Final entry point for secondary cpus from arch code, running the rest of the secondary cpu
+// init routines and entering the scheduler. Does not return.
+void lk_secondary_cpu_entry(void) __NO_RETURN;
+#endif
+
 __END_CDECLS

top/main.c

@@ -33,11 +33,6 @@ extern void (*__ctor_end[])(void);
 extern int __bss_start;
 extern int _end;
 
-#if WITH_SMP
-static thread_t *secondary_bootstrap_threads[SMP_MAX_CPUS - 1];
-static uint secondary_bootstrap_thread_count;
-#endif
-
 static int bootstrap2(void *arg);
 
 static void call_constructors(void) {
@@ -54,7 +49,7 @@ static void call_constructors(void) {
     }
 }
 
-/* called from arch code */
+//  Main C entry point of the system, called from arch code on the boot cpu.
 void lk_main(ulong arg0, ulong arg1, ulong arg2, ulong arg3) {
     // save the boot args
     lk_boot_args[0] = arg0;
@@ -135,46 +130,3 @@ static int bootstrap2(void *arg) {
 
     return 0;
 }
-
-#if WITH_SMP
-void lk_secondary_cpu_entry(void) {
-    uint cpu = arch_curr_cpu_num();
-
-    if (cpu > secondary_bootstrap_thread_count) {
-        dprintf(CRITICAL, "Invalid secondary cpu num %d, SMP_MAX_CPUS %d, secondary_bootstrap_thread_count %d\n",
-                cpu, SMP_MAX_CPUS, secondary_bootstrap_thread_count);
-        return;
-    }
-
-    thread_secondary_cpu_init_early();
-    thread_resume(secondary_bootstrap_threads[cpu - 1]);
-
-    dprintf(SPEW, "entering scheduler on cpu %d\n", cpu);
-    thread_secondary_cpu_entry();
-}
-
-static int secondary_cpu_bootstrap2(void *arg) {
-    /* secondary cpu initialize from threading level up. 0 to threading was handled in arch */
-    lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_THREADING, LK_INIT_LEVEL_LAST);
-
-    return 0;
-}
-
-void lk_init_secondary_cpus(uint secondary_cpu_count) {
-    if (secondary_cpu_count >= SMP_MAX_CPUS) {
-        dprintf(CRITICAL, "Invalid secondary_cpu_count %d, SMP_MAX_CPUS %d\n",
-                secondary_cpu_count, SMP_MAX_CPUS);
-        secondary_cpu_count = SMP_MAX_CPUS - 1;
-    }
-    for (uint i = 0; i < secondary_cpu_count; i++) {
-        dprintf(SPEW, "creating bootstrap completion thread for cpu %d\n", i + 1);
-        thread_t *t = thread_create("secondarybootstrap2",
-                                    &secondary_cpu_bootstrap2, NULL,
-                                    DEFAULT_PRIORITY, DEFAULT_STACK_SIZE);
-        t->pinned_cpu = i + 1;
-        thread_detach(t);
-        secondary_bootstrap_threads[i] = t;
-    }
-    secondary_bootstrap_thread_count = secondary_cpu_count;
-}
-#endif

top/mp.c

@@ -0,0 +1,66 @@
+// Copyright (c) 2013-2015 Travis Geiselbrecht
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+#include <assert.h>
+#include <kernel/thread.h>
+#include <lk/compiler.h>
+#include <lk/debug.h>
+#include <lk/init.h>
+#include <lk/main.h>
+
+// Top level entry points for secondary cpus
+
+#if WITH_SMP
+static thread_t *secondary_bootstrap_threads[SMP_MAX_CPUS - 1];
+static uint secondary_bootstrap_thread_count;
+
+void lk_secondary_cpu_entry_early(void) {
+    // get the cpu into threading context
+    thread_secondary_cpu_init_early();
+
+    // run early secondary cpu init routines up to the threading level
+    lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_EARLIEST, LK_INIT_LEVEL_THREADING - 1);
+}
+
+void lk_secondary_cpu_entry(void) {
+    uint cpu = arch_curr_cpu_num();
+    DEBUG_ASSERT(cpu <= secondary_bootstrap_thread_count);
+
+    thread_resume(secondary_bootstrap_threads[cpu - 1]);
+
+    dprintf(SPEW, "entering scheduler on cpu %d\n", cpu);
+    thread_secondary_cpu_entry();
+}
+
+// Secondary cpu bootstrap thread, which gives enough thread context to run the secondary cpu init routines
+// from LK_INIT_LEVEL_THREADING to LK_INIT_LEVEL_LAST.
+static int secondary_cpu_bootstrap_thread(void *arg) {
+    // Secondary cpu initialize from threading level up. 0 to threading was handled in arch
+    lk_init_level(LK_INIT_FLAG_SECONDARY_CPUS, LK_INIT_LEVEL_THREADING, LK_INIT_LEVEL_LAST);
+
+    return 0;
+}
+
+void lk_init_secondary_cpus(uint secondary_cpu_count) {
+    if (secondary_cpu_count >= SMP_MAX_CPUS) {
+        dprintf(CRITICAL, "Invalid secondary_cpu_count %d, SMP_MAX_CPUS %d\n",
+                secondary_cpu_count, SMP_MAX_CPUS);
+        secondary_cpu_count = SMP_MAX_CPUS - 1;
+    }
+    // Construct the idle and bootstrap threads for each secondary cpu
+    for (uint i = 0; i < secondary_cpu_count; i++) {
+        thread_create_secondary_cpu_idle_thread(i + 1);
+
+        dprintf(SPEW, "creating bootstrap completion thread for cpu %d\n", i + 1);
+        thread_t *t = thread_create("secondarybootstrap2",
+                                    &secondary_cpu_bootstrap_thread, NULL,
+                                    DEFAULT_PRIORITY, DEFAULT_STACK_SIZE);
+        thread_set_pinned_cpu(t, i + 1);
+        secondary_bootstrap_threads[i] = t;
+        thread_detach(t);
+    }
+    secondary_bootstrap_thread_count = secondary_cpu_count;
+}
+#endif // WITH_SMP

top/rules.mk

@@ -14,6 +14,8 @@ MODULE_SRCS := \
 	$(LOCAL_DIR)/debug.c \
 	$(LOCAL_DIR)/init.c \
 	$(LOCAL_DIR)/main.c \
+	$(LOCAL_DIR)/mp.c \
+
 
 MODULE_OPTIONS := extra_warnings