1/** 2 * \file 3 * \brief Code responsible for booting application cores 4 */ 5 6/* 7 * Copyright (c) 2007, 2008, 2009, 2010, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <stdlib.h> 16#include <stdbool.h> 17#include <stdio.h> 18#include <assert.h> 19 20#include <barrelfish/barrelfish.h> 21#include <barrelfish/cpu_arch.h> 22 23#include <if/monitor_defs.h> 24#include <if/octopus_defs.h> 25 26#include <octopus/octopus.h> 27#include <octopus/trigger.h> 28 29#include <skb/skb.h> 30#include <trace/trace.h> 31#include <barrelfish/spawn_client.h> 32 33#include <hw_records.h> 34 35#include "kaluga.h" 36 37static const char *processor_regex = HW_PROCESSOR_GENERIC_REGEX; 38 39static void cpu_change_event(octopus_mode_t mode, const char* record, void* state) 40{ 41 if (mode & OCT_ON_SET) { 42 KALUGA_DEBUG("CPU found: %s\n", record); 43 44 /* try to extract basic information from the record */ 45 uint64_t barrelfish_id, type, hw_id, enabled = 0; 46 errval_t err = oct_read(record, "_ { " HW_PROCESSOR_GENERIC_FIELDS " }", 47 &enabled, &barrelfish_id, &hw_id, &type); 48 if (err_is_fail(err)) { 49 DEBUG_ERR(err, "Cannot read record."); 50 printf("Malformed CPU record. Do not boot discovered CPU %"PRIu64".\n", 51 barrelfish_id); 52 goto out; 53 } 54 55 /* find the corectrl module for the given cpu type */ 56 struct module_info* mi = find_corectrl_for_cpu_type((enum cpu_type)type); 57 if (mi != NULL) { 58 err = mi->start_function(0, mi, (CONST_CAST)record, NULL); 59 if (err_is_fail(err)) { 60 printf("Boot driver not found. Do not boot discovered CPU %"PRIu64".\n", 61 barrelfish_id); 62 goto out; 63 } 64 } 65 } 66 if (mode & OCT_ON_DEL) { 67 KALUGA_DEBUG("CPU removed: %s\n", record); 68 assert(!"NYI"); 69 } 70 71out: 72 assert(!(mode & OCT_REMOVED)); 73} 74 75errval_t watch_for_cores(void) 76{ 77 octopus_trigger_id_t tid; 78 return oct_trigger_existing_and_watch(processor_regex, cpu_change_event, 79 NULL, &tid); 80} 81 82// State for delayed cleanup of inherit cnode 83struct inheritcn_del_st { 84 struct capref capref; 85 octopus_trigger_id_t tid; 86 coreid_t coreid; 87}; 88 89// Trigger function: gets called when spanwd on core n is up to delete 90// associated inherit cnode that was given to `corectrl boot n`. 91static void delete_inheritcn(octopus_mode_t mode, const char *record, void *state) 92{ 93 errval_t err; 94 struct inheritcn_del_st *st = state; 95 if (mode & OCT_ON_SET) { 96 KALUGA_DEBUG("spawnd up for %d: deleting inheritcn\n", st->coreid); 97 err = cap_delete(st->capref); 98 if (err_is_fail(err)) { 99 DEBUG_ERR(err, "deleting inheritcn for %d\n", st->coreid); 100 } 101 assert(err_is_ok(err)); 102 err = slot_free(st->capref); 103 if (err_is_fail(err)) { 104 DEBUG_ERR(err, "freeing slot for %d\n", st->coreid); 105 } 106 assert(err_is_ok(err)); 107 err = oct_remove_trigger(st->tid); 108 if (err_is_fail(err)) { 109 DEBUG_ERR(err, "removing trigger"); 110 } 111 assert(err_is_ok(err)); 112 free(state); 113 } 114} 115 116errval_t start_boot_driver(coreid_t where, struct module_info* mi, 117 char* record, struct driver_argument * int_arg) 118{ 119 assert(mi != NULL); 120 errval_t err; 121 122 if (!is_auto_driver(mi)) { 123 return KALUGA_ERR_DRIVER_NOT_AUTO; 124 } 125 126 // Construct additional command line arguments containing pci-id. 127 // We need one extra entry for the new argument. 128 char **argv = mi->argv; 129 bool cleanup = false; 130 char barrelfish_id_s[10]; 131 size_t argc = mi->argc; 132 133 KALUGA_DEBUG("Starting corectrl for %s\n", record); 134 uint64_t barrelfish_id, cpu_type, hw_id, enabled = 0; 135 err = oct_read(record, "_ { " HW_PROCESSOR_GENERIC_FIELDS " }", 136 &enabled, &barrelfish_id, &hw_id, &cpu_type); 137 if (err_is_ok(err)) { 138 /* 139 * XXX: change this to a generic cpuhwid instead of apic! 140 */ 141 skb_add_fact("corename(%"PRIu64", %s, apic(%"PRIu64")).", 142 barrelfish_id, cpu_type_to_archstr(cpu_type), hw_id); 143 144 /* we are already running */ 145 if (barrelfish_id == my_core_id) { 146 return SYS_ERR_OK; 147 } 148 149 if (!enabled) { 150 printf("CPU %" PRIu64 " is not enabled. Skipping driver initialization\n", 151 barrelfish_id); 152 return SYS_ERR_OK; 153 } 154 155 argv = malloc((argc+5) * sizeof(char *)); 156 memcpy(argv, mi->argv, argc * sizeof(char *)); 157 snprintf(barrelfish_id_s, 10, "%"PRIu64"", barrelfish_id); 158 159 argv[argc] = "boot"; 160 argc += 1; 161 argv[argc] = barrelfish_id_s; 162 argc += 1; 163 // Copy kernel args over to new core 164 struct module_info* cpu_module = find_module("cpu"); 165 if (cpu_module != NULL && strlen(cpu_module->args) > 1) { 166 KALUGA_DEBUG("%s:%s:%d: Boot with cpu arg %s and barrelfish_id_s=%s\n", 167 __FILE__, __FUNCTION__, __LINE__, cpu_module->args, barrelfish_id_s); 168 argv[argc] = "-a"; 169 argc += 1; 170 argv[argc] = cpu_module->args; 171 argc += 1; 172 } 173 argv[argc] = NULL; 174 175 cleanup = true; 176 } 177 else { 178 DEBUG_ERR(err, "Malformed CPU record?"); 179 return err; 180 } 181 182 struct capref task_cap_kernel; 183 task_cap_kernel.cnode = cnode_task; 184 task_cap_kernel.slot = TASKCN_SLOT_KERNELCAP; 185 186#ifdef KALUGA_SERVICE_DEBUG 187 struct capability info; 188 err = debug_cap_identify(task_cap_kernel, &info); 189 if (err_is_fail(err)) { 190 USER_PANIC_ERR(err, "Can not identify the capability."); 191 } 192 char buffer[1024]; 193 debug_print_cap(buffer, 1024, &info); 194 KALUGA_DEBUG("%s:%d: capability=%s\n", __FILE__, __LINE__, buffer); 195#endif 196 197 struct capref inheritcn_cap; 198 err = alloc_inheritcn_with_caps(&inheritcn_cap, 199 NULL_CAP, NULL_CAP, task_cap_kernel); 200 if (err_is_fail(err)) { 201 DEBUG_ERR(err, "alloc_inheritcn_with_caps failed."); 202 } 203 204 err = spawn_program_with_caps(where, mi->path, argv, 205 environ, inheritcn_cap, 206 NULL_CAP, SPAWN_FLAGS_NEW_DOMAIN, 207 &mi->did[0]); 208 if (err_is_fail(err)) { 209 DEBUG_ERR(err, "Spawning %s failed.", mi->path); 210 } 211 212 if (cleanup) { 213 free(argv); 214 } 215 216 // Cannot just delete inherit cnode capability here, as deleting any CNode 217 // copy will always delete all copies of the CNode. Store inherit cnode 218 // cap in octopus, and delete it when the matching spawnd is up 219 char spawnd[32]; 220 snprintf(spawnd, 32, "spawn.%s", barrelfish_id_s); 221 struct inheritcn_del_st *tstate = calloc(1, sizeof(*tstate)); 222 tstate->capref = inheritcn_cap; 223 tstate->coreid = barrelfish_id; 224 errval_t err2 = oct_trigger_existing_and_watch(spawnd, delete_inheritcn, 225 tstate, &tstate->tid); 226 if (err_is_fail(err2)) { 227 free(tstate); 228 return err2; 229 } 230 231 return err; 232} 233 234 235static void spawnd_change_event(octopus_mode_t mode, const char* record, 236 void* state) 237{ 238 size_t count = (size_t) state; 239 static coreid_t spawnd_counter = 0; 240 241 if (mode & OCT_ON_SET) { 242 KALUGA_DEBUG("spawnd found: %s\n", record); 243 spawnd_counter++; 244 245 if (spawnd_counter == count) { 246 KALUGA_DEBUG("Found enough spawnds, setting all_spawnds_up\n"); 247 errval_t err = oct_set("all_spawnds_up { iref: 0 }"); 248 assert(err_is_ok(err)); 249 } 250 } 251} 252 253extern size_t cpu_count; 254 255errval_t wait_for_all_spawnds(void) 256{ 257 // Note: The whole wait for all_spawnds_up thing is a hack. 258 // Our overall design goal is a system where cores 259 // come and go dynamically and we do not want / need 260 // to wait for a stable state. 261 // However, some of our code (for example domain spanning) 262 // still assumes a fixed set of cores and will deadlock 263 // otherwise. Therefore we need to fix those parts first. 264 errval_t err; 265 char* record = NULL; 266#if !defined(__ARM_ARCH_7A__) 267 KALUGA_DEBUG("Waiting for acpi"); 268 err = oct_wait_for(&record, "acpi { iref: _ }"); 269 if (err_is_fail(err)) { 270 return err_push(err, KALUGA_ERR_WAITING_FOR_ACPI); 271 } 272#endif 273 274 // No we should be able to get core count 275 // of all cores to estimate the amount of 276 // spawnd's we have to expect (one per core) 277 char** names; 278 size_t count; 279 err = oct_get_names(&names, &count, processor_regex); 280 if (err_is_fail(err)) { 281 return err_push(err, KALUGA_ERR_QUERY_LOCAL_APIC); 282 } 283 oct_free_names(names, count); 284 285 if (cpu_count) { 286 count = cpu_count; 287 } 288 289 static char* spawnds = "r'spawn.[0-9]+' { iref: _ }"; 290 octopus_trigger_id_t tid; 291 err = oct_trigger_existing_and_watch(spawnds, spawnd_change_event, (void*)count, &tid); 292 if (err_is_fail(err)) { 293 return err_push(err, KALUGA_ERR_QUERY_LOCAL_APIC); 294 } 295 296 return oct_wait_for(&record, "all_spawnds_up { iref: 0 }"); 297} 298