1/** 2 * \file 3 * \brief Driver for booting the Xeon Phi Coprocessor card on a Barrelfish Host 4 */ 5 6/* 7 * Copyright (c) 2014 ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <stdio.h> 16#include <string.h> 17#include <barrelfish/barrelfish.h> 18#include <barrelfish/dispatch.h> 19#include <barrelfish/waitset.h> 20#include <barrelfish/nameservice_client.h> 21#include <barrelfish/spawn_client.h> 22#include <octopus/octopus.h> 23 24#include <xeon_phi/xeon_phi.h> 25 26#include "xeon_phi_internal.h" 27#include "interphi.h" 28#include "dma_service.h" 29#include "sysmem_caps.h" 30#include "smpt.h" 31#include "xphi_service.h" 32 33static struct xeon_phi xphi; 34 35static struct capref mmio_cap = { 36 .slot = TASKCN_SLOT_IO 37}; 38 39static struct capref sysmem_cap = { 40 .slot = TASKCN_SLOT_SYSMEM 41}; 42 43static struct capref host_cap; 44 45static errval_t map_mmio_space(struct xeon_phi *phi) 46{ 47 errval_t err; 48 void *mmio; 49 50 struct frame_identity id; 51 err = invoke_frame_identify(mmio_cap, &id); 52 if (err_is_fail(err)) { 53 return err; 54 } 55 56 err = vspace_map_one_frame(&mmio, id.bytes, mmio_cap, NULL, NULL); 57 if (err_is_fail(err)) { 58 return err; 59 } 60 61 XDEBUG("mapped mmio register space @ [%p]\n", mmio); 62 63 phi->mmio.vbase = (lvaddr_t) mmio; 64 phi->mmio.cap = mmio_cap; 65 phi->mmio.pbase = id.base; 66 phi->mmio.length = id.bytes; 67 68 return SYS_ERR_OK; 69} 70 71/* 72 * ------------------------------------------------------------------------------- 73 * event loop 74 * ------------------------------------------------------------------------------- 75 */ 76 77/** 78 * \brief handles events on the waitset and polls for completed DMA transfers 79 * and new data on the serial line (host only) 80 * 81 * \param do_yield if set, yield thread if no event was discovered 82 * 83 * \return SYS_ERR_OK if an event was handled 84 * LIB_ERR_NO_EVENT if there was no evetn 85 */ 86errval_t xeon_phi_event_poll(uint8_t do_yield) 87{ 88 errval_t err; 89 90 uint8_t idle = 0x1; 91 err = xdma_service_poll(&xphi); 92 switch(err_no(err)) { 93 case SYS_ERR_OK: 94 idle = 0; 95 break; 96 case DMA_ERR_DEVICE_IDLE: 97 /* no op */ 98 break; 99 default: 100 return err; 101 break; 102 } 103 err = event_dispatch_non_block(get_default_waitset()); 104 switch(err_no(err)) { 105 case LIB_ERR_NO_EVENT : 106 if (idle) { 107 if (do_yield) { 108 thread_yield(); 109 return SYS_ERR_OK; 110 } else { 111 return LIB_ERR_NO_EVENT; 112 } 113 } else { 114 return SYS_ERR_OK; 115 } 116 default: 117 return err; 118 break; 119 } 120 return SYS_ERR_OK; 121 122} 123 124/* 125 * ------------------------------------------------------------------------------- 126 * booting cores 127 * ------------------------------------------------------------------------------- 128 */ 129 130static char *cores_arg = NULL; 131coreid_t cores_count = 0; 132coreid_t cores_seen = 0; 133 134#define USE_OCTOPUS_EVENTS 0 135#if USE_OCTOPUS_EVENTS 136 137 138 139 140octopus_trigger_id_t cores_tid; 141 142static void spawnd_change_event(octopus_mode_t mode, char* record, void* state) 143{ 144 debug_printf("spawnd_change_event...\n"); 145 if (mode & OCT_ON_SET) { 146 debug_printf("spawnd found: %s\n", record); 147 cores_seen++; 148 149 if (cores_seen == cores_count) { 150 errval_t err = oct_set("all_spawnds_up { iref: 0 }"); 151 assert(err_is_ok(err)); 152 err = oct_remove_trigger(cores_tid); 153 assert(err_is_ok(err)); 154 } 155 } 156} 157 158#endif 159 160static errval_t boot_cores(void) 161{ 162 errval_t err; 163 164 debug_printf("booting cores...\n"); 165 166 char *arg[4]; 167 arg[0] = "corectrl"; 168 arg[1] = "boot"; 169 170 /* spawn core boot */ 171 if (cores_arg != NULL) { 172 arg[2] = cores_arg; 173 } else { 174 arg[2] = "1:9"; 175 cores_count = 10; 176 177 } 178 arg[3] = NULL; 179 180#if USE_OCTOPUS_EVENTS 181 debug_printf("setting trigger...\n"); 182 183 /* set trigger for booting cors */ 184 err = oct_trigger_existing_and_watch("r'spawn.[0-9]+' { iref: _ }", 185 spawnd_change_event, &cores_count, 186 &cores_tid); 187 if (err_is_fail(err)) { 188 /* TODO: ERROR HANDLING */ 189 } 190#endif 191 debug_printf("spawning corectrl...\n"); 192 193 struct capref new_domain; 194 struct capref coreboot_cap = {cnode_task, TASKCN_SLOT_COREBOOT}; 195 196 /* Create argument cnode to pass coreboot cap */ 197 struct capref argcn_cap, coreboot_cap_in_argcn; 198 err = cnode_create_l2(&argcn_cap, &coreboot_cap_in_argcn.cnode); 199 assert(err_is_ok(err)); 200 coreboot_cap_in_argcn.slot = 0; 201 err = cap_copy(coreboot_cap_in_argcn, coreboot_cap); 202 assert(err_is_ok(err)); 203 204 // Wait until spawnd 0 is up (since we go over proc_mgmt 205 // that might throw a error if spawnd 0 is not up) 206 err = nameservice_blocking_lookup("spawn.0", NULL); 207 assert(err_is_ok(err)); 208 209 err = spawn_program_with_caps(0, "k1om/sbin/corectrl", arg, NULL, NULL_CAP, 210 argcn_cap, 0, &new_domain); 211 assert(err_is_ok(err)); 212 213 /* Delete our copy of argument cnode */ 214 err = cap_destroy(argcn_cap); 215 assert(err_is_ok(err)); 216 217#if !USE_OCTOPUS_EVENTS 218 debug_printf("waiting for spawn...\n"); 219 for (cores_seen = 0; cores_seen < cores_count; ++cores_seen) { 220 char buf[10]; 221 debug_printf("XX waiting for spawn.%u\n", cores_seen); 222 snprintf(buf, 10, "spawn.%u", cores_seen); 223 err = nameservice_blocking_lookup(buf, NULL); 224 assert(err_is_ok(err)); 225 } 226 227 err = oct_set("all_spawnds_up { iref: 0 }"); 228 assert(err_is_ok(err)); 229#endif 230 231 return SYS_ERR_OK; 232} 233 234static void parse_arguments(int argc, char** argv) 235{ 236 for (int i = 1; i < argc; i++) { 237 if (strncmp(argv[i], "cpus=", 5) == 0) { 238 cores_arg = argv[i] + 5; 239 } 240 } 241} 242 243/* 244 * ------------------------------------------------------------------------------- 245 * Main 246 * ------------------------------------------------------------------------------- 247 */ 248 249int main(int argc, 250 char *argv[]) 251{ 252 debug_printf("Xeon Phi module started on node [%u].\n", disp_xeon_phi_id()); 253 254 errval_t err; 255 256 mmio_cap.cnode = cnode_task; 257 sysmem_cap.cnode = cnode_task; 258 259 assert(!capref_is_null(mmio_cap)); 260 assert(!capref_is_null(sysmem_cap)); 261 262 xphi.is_client = 0x1; 263 xphi.id = disp_xeon_phi_id(); 264 265 for (uint32_t i = 0; i < XEON_PHI_NUM_MAX; ++i) { 266 xphi.topology[i].id = i; 267 xphi.topology[i].local = &xphi; 268 } 269 270 parse_arguments(argc, argv); 271 272 XDEBUG("Initializing system memory cap manager...\n"); 273 err = sysmem_cap_manager_init(sysmem_cap); 274 if (err_is_fail(err)) { 275 USER_PANIC_ERR(err, "Could not initialize the cap manager.\n"); 276 } 277 278 err = map_mmio_space(&xphi); 279 if (err_is_fail(err)) { 280 USER_PANIC_ERR(err, "could not map the mmio space"); 281 } 282 283 err = oct_init(); 284 if (err_is_fail(err)) { 285 USER_PANIC_ERR(err, "Octopus initialization failed."); 286 } 287 288 /* boot cores */ 289 err = boot_cores(); 290 if (err_is_fail(err)) { 291 USER_PANIC_ERR(err, "spawning coreboot\n"); 292 } 293 294 /* wait until the kernels are booted and spawnds are ready */ 295 err = nameservice_blocking_lookup("all_spawnds_up", NULL); 296 if (err_is_fail(err)) { 297 USER_PANIC_ERR(err, "all_spawnds_up.\n"); 298 } 299 300 err = xdma_service_init(&xphi); 301 if (err_is_fail(err)) { 302 USER_PANIC_ERR(err, "Could not initialize the dma engine.\n"); 303 } 304 305 err = smpt_init(&xphi); 306 if (err_is_fail(err)) { 307 USER_PANIC_ERR(err, "Could not initialize the SMTP.\n"); 308 } 309 310 lpaddr_t host_msg_base = strtol(argv[0], NULL, 16); 311 uint8_t host_msg_size = strtol(argv[1], NULL, 16); 312 313 XMESSAGING_DEBUG("Getting the host messaging cap...[%016lx, %02x]\n", 314 host_msg_base, host_msg_size); 315 316 err = sysmem_cap_request(host_msg_base, host_msg_size, &host_cap); 317 if (err_is_fail(err)) { 318 USER_PANIC_ERR(err, "Could not obtain the system messsaging cap\n"); 319 } 320 321 err = interphi_init(&xphi, host_cap); 322 if (err_is_fail(err)) { 323 USER_PANIC_ERR(err, "Could not initialize the interphi communication\n"); 324 } 325 326 err = xeon_phi_service_init(&xphi); 327 if (err_is_fail(err)) { 328 USER_PANIC_ERR(err, "could not initialize the messaging service"); 329 } 330 331 XDEBUG("Start polling for messages...\n"); 332 333 while (1) { 334 xeon_phi_event_poll(1); 335 } 336 337 XDEBUG("Messaging loop terminated...\n"); 338 return 0; 339} 340