1/** 2 * \file 3 * \brief Boot module for the Xeon Phi 4 * 5 * Loads the co processor OS onto the card and boots it 6 */ 7 8/* 9 * Copyright (c) 2014 ETH Zurich. 10 * All rights reserved. 11 * 12 * This file is distributed under the terms in the attached LICENSE file. 13 * If you do not find this file, copies can be found by writing to: 14 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 15 */ 16 17#include <stdio.h> 18#include <string.h> 19#include <ctype.h> 20#include <sys/param.h> 21 22#include <barrelfish/barrelfish.h> 23#include <spawndomain/spawndomain.h> 24#include <vfs/vfs.h> 25#include <elf/elf.h> 26 27#include <xeon_phi/xeon_phi.h> 28#include <tftp/tftp.h> 29 30#include <multiboot.h> 31 32#include <dev/xeon_phi/xeon_phi_boot_dev.h> 33#include <dev/xeon_phi/xeon_phi_apic_dev.h> 34 35struct bootinfo *bi = NULL; 36 37#include "xeon_phi_internal.h" 38#include "interphi.h" 39#include "sleep.h" 40 41#define TFTP_BUF_SIZE (1<<20) 42#define TFTP_BUF_SIZE_MODULES (1<<30) 43 44typedef errval_t (*loadfile_fn_t)(char *path, void *buf, size_t buflen, size_t *ret_size); 45 46#define BOOT_TIMEOUT 3000 47#define BOOT_COUNTER 0xFFFFF 48 49/* 50 * TODO: Verify these values if they are really needed 51 */ 52#define MEMORY_RESERVE_PERCENT 50 53#define UOS_RESERVE_SIZE_MIN ((128) * 1024 * 1024) 54#define UOS_RESERVE_SIZE_MAX (((4) * 1024 * 1024 * 1024ULL) - ((4) * 1024)) 55 56#define ALIGN_BASE_PAGE(x) ROUND_UP(x, BASE_PAGE_SIZE) 57 58static xeon_phi_boot_t boot_registers; 59static xeon_phi_apic_t apic_registers; 60 61/** 62 * \brief get the load offset to where to place the bootloader 63 * 64 * The bootstrap on the card will write the offset into the SBOX_SCRATCH2 65 * register once the bootstrap is finished 66 */ 67static inline lvaddr_t get_load_offset(struct xeon_phi *phi) 68{ 69 return ((lvaddr_t) xeon_phi_boot_download_offset_rdf(&boot_registers)) << 12; 70} 71 72static uint64_t get_adapter_memsize(void) 73{ 74 xeon_phi_boot_meminfo_t meminfo = xeon_phi_boot_meminfo_rd(&boot_registers); 75 76 uint64_t memsize = xeon_phi_boot_meminfo_size_kb_extract(meminfo); 77 memsize *= 1024; 78 79 switch (xeon_phi_boot_meminfo_usage_extract(meminfo)) { 80 case xeon_phi_boot_mem_all: 81 return memsize; 82 case xeon_phi_boot_mem_half: 83 return (memsize / 2); 84 case xeon_phi_boot_mem_third: 85 return (memsize / 3); 86 break; 87 case xeon_phi_boot_mem_fourth: 88 return (memsize / 4); 89 default: 90 return memsize; 91 } 92} 93 94/** 95 * \brief generates the cmdline supplied to the card kernel 96 * 97 * \param phi the card information structure 98 * \param load_offset offset where to load the cmdline 99 * \param ret_size size of the cmdline in bytes 100 */ 101static errval_t load_cmdline(struct xeon_phi *phi, lvaddr_t load_offset) 102{ 103 uint32_t cmdlen = 0; 104 105 struct xeon_phi_boot_params *bp; 106 bp = (struct xeon_phi_boot_params *)(phi->apt.vbase + phi->os_offset); 107 108 void *buf = (void *) (phi->apt.vbase + load_offset); 109 110 if (phi->cmdline) { 111 cmdlen += sprintf(buf + cmdlen, "%s", phi->cmdline); 112 } 113 114 cmdlen += sprintf(buf + cmdlen, "card_id=%i", phi->id); 115 116 /* 117 * id 118 */ 119 /* 120 * TODO: Add multihop / communication information here.. 121 */ 122 123 XBOOT_DEBUG("cmdline @ 0x%" PRIx32 " '%s'\n", (uint32_t)load_offset, (char*)buf); 124 125 phi->cmdline = buf; 126 phi->cmdlen = cmdlen; 127 128 129 bp->cmdline_ptr = (uint32_t)(load_offset); 130 bp->cmdline_size = (uint32_t)cmdlen; 131 132 return SYS_ERR_OK; 133} 134 135static errval_t bootstrap_notify(struct xeon_phi *phi) 136{ 137 // set the bootimage size to tell the bootloader 138 xeon_phi_boot_os_size_rawwr(&boot_registers, phi->os_size); 139 140 uint64_t memsize = get_adapter_memsize(); 141 142 uint64_t reserved = (memsize * MEMORY_RESERVE_PERCENT / 100); 143 144 // Keep in mind maximum uos reserve size is uint32_t, so we never overflow 145 reserved = MIN(reserved, UOS_RESERVE_SIZE_MAX); 146 reserved = MAX(reserved, UOS_RESERVE_SIZE_MIN); 147 148 // Always align uos reserve size to a page 149 reserved = (reserved & ~(BASE_PAGE_SIZE - 1)); 150 151 xeon_phi_boot_res_size_rawwr(&boot_registers, (uint32_t) reserved); 152 153 // sending the bootstrap interrupt 154 xeon_phi_apic_icr_lo_t icr_lo = xeon_phi_apic_icr_lo_default; 155 icr_lo = xeon_phi_apic_icr_lo_vector_insert(icr_lo, xeon_phi_apic_vec_bsp); 156 icr_lo = xeon_phi_apic_icr_lo_boot_notify_insert(icr_lo, 0x1); 157 158 assert(icr_lo == (229 | (1 << 13))); 159 160 xeon_phi_apic_icr_hi_wr(&apic_registers, xeon_phi_apic_bootstrap, phi->apicid); 161 162 xeon_phi_apic_icr_lo_wr(&apic_registers, xeon_phi_apic_bootstrap, icr_lo); 163 164 return SYS_ERR_OK; 165} 166 167/* 168 * ------------------------------------------------------------------------------- 169 * VFS helper function 170 * ------------------------------------------------------------------------------- 171 */ 172 173static vfs_handle_t file_open(char *file, uint8_t nfs) 174{ 175 errval_t err; 176 177 vfs_handle_t fh; 178 char *path = file; 179 if (nfs) { 180 size_t path_size = strlen(file) + strlen(XEON_PHI_NFS_MNT) + 2; 181 path = malloc(path_size); 182 if (path == NULL) { 183 return NULL; 184 } 185 if (file[0] == '/') { 186 snprintf(path, path_size, "%s%s", XEON_PHI_NFS_MNT, file); 187 } else { 188 snprintf(path, path_size, "%s/%s", XEON_PHI_NFS_MNT, file); 189 } 190 } 191 192 err = vfs_open(path, &fh); 193 if (nfs) { 194 free(path); 195 } 196 switch(err_no(err)) { 197 case SYS_ERR_OK : 198 return fh; 199 break; 200 case FS_ERR_NOTFOUND : 201 err = vfs_open(file, &fh); 202 if (err_is_ok(err)) { 203 return fh; 204 } 205 return NULL; 206 break; 207 default: 208 return NULL; 209 } 210 211 return NULL; 212} 213 214static errval_t file_load(vfs_handle_t fh, void *buf, size_t length) 215{ 216 errval_t err; 217 size_t pos = 0, readlen = 0; 218 do { 219 err = vfs_read(fh, buf+pos, length - pos, &readlen); 220 if (err_is_fail(err)) { 221 return err; 222 } else if (readlen == 0) { 223 return SPAWN_ERR_LOAD; // XXX 224 } else { 225 pos += readlen; 226 } 227 } while (err_is_ok(err) && readlen > 0 && pos < length); 228 229 return SYS_ERR_OK; 230} 231 232 233static errval_t download_file_vfs(char *file, void *card_buffer, size_t use_nfs, 234 size_t *bytes) 235{ 236 errval_t err; 237 238 vfs_handle_t fh = file_open(file, use_nfs); 239 if (fh == NULL) { 240 return FS_ERR_INVALID_FH; 241 } 242 243 struct vfs_fileinfo info; 244 err = vfs_stat(fh, &info); 245 if (err_is_fail(err)) { 246 goto out; 247 } 248 249 if (bytes) { 250 *bytes = info.size; 251 } 252 253 assert(info.type == VFS_FILE); 254 assert(info.size < 4UL<<30); 255 256 err = file_load(fh, card_buffer, info.size); 257 out: 258 vfs_close(fh); 259 return err; 260} 261 262/* 263 * ------------------------------------------------------------------------------- 264 * boot loader 265 * ------------------------------------------------------------------------------- 266 */ 267 268#define SETUP_SECTORS 2 269#define SECTOR_SIZE 512 270#define HEADER_SIZE (SETUP_SECTORS*SECTOR_SIZE) 271 272static errval_t download_bootloader_generic(struct xeon_phi *phi, char *bootloader, 273 loadfile_fn_t loadfn, size_t args) 274{ 275 errval_t err; 276 277 lvaddr_t loadoffset = get_load_offset(phi); 278 size_t imgsize; 279 280 char *buf = (void *) (phi->apt.vbase + loadoffset); 281 282 /* fill in the header */ 283 memset(buf, 0, HEADER_SIZE); 284 285 /* 286 * This is the signature. Without this the kernel does not boot. 287 * Signature is reads "HdrS" 288 */ 289 buf[514] = 0x48; 290 buf[515] = 0x64; 291 buf[516] = 0x72; 292 buf[517] = 0x53; 293 buf[0x1f1] = SETUP_SECTORS-1; 294 295 XBOOT_DEBUG("loading %s @ 0x%lx\n", bootloader, loadoffset); 296 297 err = loadfn(bootloader, buf + HEADER_SIZE, args, &imgsize); 298 if (err_is_fail(err)) { 299 return err; 300 } 301 302 size_t sys_size = (imgsize + 15) / 16; 303 buf[0x1f4] = sys_size; 304 buf[0x1f5] = sys_size >> 8; 305 buf[0x1f6] = sys_size >> 16; 306 buf[0x1f7] = sys_size >> 24; 307 308 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers); 309 310 phi->os_offset = loadoffset; 311 phi->os_size = imgsize; 312 313 XBOOT_DEBUG("Xeon Phi bootloader %s loaded @ 0x%lx size %lu kB\n", bootloader, 314 loadoffset, imgsize >> 10); 315 316 return SYS_ERR_OK; 317} 318 319 320static inline errval_t download_bootloader_vfs(struct xeon_phi *phi, char *bootloader, 321 uint8_t use_nfs) 322{ 323 return download_bootloader_generic(phi, bootloader, download_file_vfs, use_nfs); 324} 325 326static inline errval_t download_bootloader_tftp(struct xeon_phi *phi, char *bootloader) 327{ 328 return download_bootloader_generic(phi, bootloader, tftp_client_read_file, 329 TFTP_BUF_SIZE_MODULES); 330} 331 332 333/* 334 * ------------------------------------------------------------------------------- 335 * multiboot modules 336 * ------------------------------------------------------------------------------- 337 */ 338 339static uint32_t prepare_multiboot_strings(void *strings, char **mods, 340 uint32_t num_mods) 341{ 342 uint32_t bytes = 0; 343 for (uint32_t i = 0; i < num_mods; ++i) { 344 bytes += snprintf(strings+bytes, 1<<20, "%s", mods[i]) + 1; 345 } 346 return bytes; 347} 348 349static inline char *get_module_path(char *cmdline) 350{ 351 while(*cmdline) { 352 if (isspace((int)*cmdline)) { 353 return cmdline; 354 } 355 cmdline++; 356 } 357 return cmdline; 358} 359 360static uint32_t prepare_multiboot_info(void *aptvbase, lpaddr_t offset, 361 char **mmaps, uint32_t num_mods, 362 uint32_t num_mmaps) 363{ 364 void *mbibuf = aptvbase + offset; 365 /* 366 * Layout of multi boot information on card: 367 * [multiboot_info] 368 * [n x multiboot_modinfo] 369 * [m x multiboot_mmap] 370 * [strings] 371 */ 372 373 /* set the host virtual pointers of the multiboot structures */ 374 struct multiboot_info *mbi = mbibuf; 375 struct multiboot_modinfo *mbi_mods = (struct multiboot_modinfo *)(mbi + 1); 376 struct multiboot_mmap *mbi_mmaps = (struct multiboot_mmap *)(mbi_mods + num_mods); 377 378 /* card physical modules array */ 379 offset += (uint32_t)sizeof(struct multiboot_info); 380 mbi->mods_count = num_mods; 381 mbi->mods_addr = offset; 382 383 /* card physical mmap array */ 384 offset += num_mods * (uint32_t)sizeof(struct multiboot_modinfo); 385 mbi->mmap_addr = offset; 386 mbi->mmap_length = num_mmaps * sizeof(struct multiboot_mmap); 387 388 offset += num_mmaps * (uint32_t)sizeof(struct multiboot_mmap); 389 390 /* set the multiboot flags */ 391 mbi->flags = MULTIBOOT_INFO_FLAG_HAS_CMDLINE | MULTIBOOT_INFO_FLAG_HAS_MODS 392 | MULTIBOOT_INFO_FLAG_HAS_ELF_SYMS| MULTIBOOT_INFO_FLAG_HAS_MMAP; 393 394 for (uint32_t i = 0; i < num_mmaps; ++i) { 395 uint32_t parsed = sscanf(mmaps[i],"map%*[ \n\t]%" SCNx64 396 "%*[ \n\t]%" SCNx64 "%*[ \n\t]%" SCNu32, 397 &mbi_mmaps[i].base_addr, 398 &mbi_mmaps[i].length, 399 &mbi_mmaps[i].type); 400 if (parsed !=3) { 401 debug_printf("INVALID mmap: {%s}\n", mmaps[i]); 402 mbi_mmaps[i].size = 0; 403 continue; 404 } 405 mbi_mmaps[i].size = sizeof(struct multiboot_mmap); 406 } 407 408 return sizeof(*mbi) + num_mods * sizeof(*mbi_mods) + num_mmaps * sizeof(*mbi_mmaps); 409} 410 411static errval_t download_modules_generic(struct xeon_phi *phi, size_t offset, 412 char **mods, uint32_t num_mods, 413 uint32_t num_mmaps, loadfile_fn_t loadfn, 414 size_t args) 415{ 416 errval_t err; 417 418 struct xeon_phi_boot_params *bp; 419 bp = (struct xeon_phi_boot_params *)(phi->apt.vbase + phi->os_offset); 420 421 struct multiboot_info *mbi = (void *)phi->apt.vbase + offset; 422 struct multiboot_modinfo *mbi_mods = (struct multiboot_modinfo *)(mbi + 1); 423 424 bp->ramdisk_image = offset; 425 bp->mbi = offset; 426 427 offset += prepare_multiboot_info((void *)phi->apt.vbase, offset, mods + num_mods, 428 num_mods, num_mmaps); 429 430 lpaddr_t strings_offset = offset; 431 offset += prepare_multiboot_strings((void *)phi->apt.vbase + strings_offset, 432 mods, num_mods); 433 434 offset = ALIGN_BASE_PAGE(offset); 435 436 for (uint32_t i = 0; i < num_mods; ++i) { 437 char *strings = (void *)phi->apt.vbase + strings_offset; 438 size_t cmdlength = strlen(strings); 439 size_t imgsize = 0; 440 mbi_mods[i].mod_start = offset; 441 mbi_mods[i].string = strings_offset; 442 443 char *delim = get_module_path(mods[i]); 444 *delim = 0; 445 446 err = loadfn(mods[i], (void *)phi->apt.vbase + offset, args, &imgsize); 447 if (err_is_fail(err)) { 448 return err; 449 } 450 mbi_mods[i].mod_end = mbi_mods[i].mod_start + imgsize; 451 452 offset = ALIGN_BASE_PAGE(offset + imgsize); 453 454 XBOOT_DEBUG("module %35s @ 0x08%lx size %lu kB\n", 455 (char *)phi->apt.vbase + strings_offset, offset, imgsize >> 10); 456 457 strings_offset += (cmdlength + 1); 458 } 459 460 bp->ramdisk_size = offset - bp->ramdisk_image; 461 462 return SYS_ERR_OK; 463} 464 465 466static errval_t download_modules_vfs(struct xeon_phi *phi, size_t offset, 467 char **mods, uint32_t num_mods, 468 uint32_t num_mmaps, uint8_t use_nfs) 469{ 470 return download_modules_generic(phi, offset, mods, num_mods, num_mmaps, 471 download_file_vfs, use_nfs); 472} 473 474static errval_t download_modules_tftp(struct xeon_phi *phi, lpaddr_t offset, 475 char **mods, uint32_t num_mods, 476 uint32_t num_mmaps) 477{ 478 return download_modules_generic(phi, offset, mods, num_mods, num_mmaps, 479 tftp_client_read_file, TFTP_BUF_SIZE_MODULES); 480} 481 482/* 483 * ------------------------------------------------------------------------------- 484 * Parsing modules 485 * ------------------------------------------------------------------------------- 486 */ 487 488static inline char *discard_leading_white_spaces(char *string) 489{ 490 while(*string) { 491 if (!isspace((int)*string)) { 492 break; 493 } 494 string++; 495 } 496 return string; 497} 498 499 500static errval_t parse_mod_list(char *modules, uint32_t *mods, uint32_t *mmaps, 501 uint8_t *kernel, char ***parsed_modules) 502{ 503 uint32_t num_mod = 0, num_mmap = 0; 504 uint8_t has_kernel = 0; 505 506 char *line = modules; 507 508 /* how many modules we have */ 509 while (line != NULL) 510 { 511 if (*line == '\n') { 512 line++; 513 } 514 if (strncmp(line, "module", 6)==0) { 515 num_mod ++; 516 } else if (strncmp(line, "kernel", 6) == 0) { 517 assert(has_kernel == 0); 518 has_kernel = 1; 519 } else if (strncmp(line, "mmap", 4) == 0) { 520 num_mmap++; 521 } 522 line=strchr(line+1,'\n'); 523 } 524 525 526 /* allocate parsed array */ 527 char **parsed = calloc(num_mod + num_mmap + 1, sizeof(char *)); 528 if (parsed == NULL) { 529 return LIB_ERR_MALLOC_FAIL; 530 } 531 532 uint32_t mod_idx = 1; 533 uint32_t mmap_idx = num_mod + num_mmap; 534 line = modules; 535 while (line != NULL) 536 { 537 if (*line == '\n') { 538 *line = 0; 539 line++; 540 } 541 if (strncmp(line, "module", 6)==0) { 542 parsed[mod_idx++] = discard_leading_white_spaces(line + 6); 543 } else if (strncmp(line, "kernel", 6) == 0) { 544 parsed[0] = discard_leading_white_spaces(line + 6); 545 } else if (strncmp(line, "mmap", 4) == 0) { 546 parsed[mmap_idx--] = discard_leading_white_spaces(line + 4); 547 } 548 line=strchr(line+1,'\n'); 549 } 550 551 if (parsed_modules) { 552 *parsed_modules = parsed; 553 } else { 554 free(parsed_modules); 555 } 556 557 if (mods) { 558 *mods = num_mod; 559 } 560 561 if (mmaps) { 562 *mmaps = num_mmap; 563 } 564 565 if (kernel) { 566 *kernel = has_kernel; 567 } 568 569 XBOOT_DEBUG("parsing modules found: %u kernel, %u modules, %u mmaps\n", 570 has_kernel, num_mod, num_mmap); 571 572 return SYS_ERR_OK; 573} 574 575 576static errval_t load_mod_list_tftp(char *mod_list, void **modules, size_t *size) 577{ 578 errval_t err; 579 580 void *buf = calloc(1, TFTP_BUF_SIZE); 581 582 XBOOT_DEBUG("loading modules list %s over TFTP\n", mod_list); 583 584 err = tftp_client_read_file(mod_list, buf, TFTP_BUF_SIZE, size); 585 if (err_is_fail(err)) { 586 USER_PANIC("reading tftp file"); 587 } 588 589 if (modules) { 590 *modules = buf; 591 } 592 593 return SYS_ERR_OK; 594} 595 596static errval_t load_mod_list_vfs(char *mod_list, uint8_t use_nfs, 597 void **modules, size_t *size) 598{ 599 errval_t err; 600 601 XBOOT_DEBUG("loading modules list %s %s\n", mod_list, 602 (use_nfs==1 ? "over NFS" : "from ramfs")); 603 604 /* load the menu lst file */ 605 vfs_handle_t fh = file_open(mod_list, use_nfs); 606 if (fh == NULL) { 607 return SPAWN_ERR_LOAD; 608 } 609 610 struct vfs_fileinfo info; 611 err = vfs_stat(fh, &info); 612 if (err_is_fail(err)) { 613 vfs_close(fh); 614 return err_push(err, SPAWN_ERR_LOAD); 615 } 616 617 assert(info.type == VFS_FILE); 618 619 char *menulst = calloc(info.size + 1, 1); 620 if (menulst == NULL) { 621 vfs_close(fh); 622 return LIB_ERR_MALLOC_FAIL; 623 } 624 625 err = file_load(fh, menulst, info.size); 626 if (err_is_fail(err)) { 627 USER_PANIC_ERR(err, "file loading failed.\n"); 628 vfs_close(fh); 629 free(menulst); 630 return err; 631 } 632 633 if (modules) { 634 *modules = menulst; 635 } 636 637 if (size) { 638 *size = info.size; 639 } 640 641 return SYS_ERR_OK; 642} 643 644/** 645 * \brief boots the card with the given loader and multiboot image 646 * 647 * \param phi pointer to the card information 648 * \param mod_uri name of to the modules location uri 649 * \param mod_list name of the modules list 650 */ 651errval_t xeon_phi_boot(struct xeon_phi *phi, 652 char *mod_uri, 653 char *mod_list) 654{ 655 errval_t err; 656 lvaddr_t offset; 657 658 xeon_phi_boot_initialize(&boot_registers, 659 XEON_PHI_MMIO_TO_SBOX(phi), 660 XEON_PHI_MMIO_TO_DBOX(phi)); 661 xeon_phi_apic_initialize(&apic_registers, XEON_PHI_MMIO_TO_SBOX(phi)); 662 663 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers); 664 665 void *modules = NULL; 666 size_t modules_size = 0; 667 uint8_t use_nfs = 0, use_tftp = 0; 668 if (strncmp(mod_uri, "nfs://", 6) == 0) { 669 XBOOT_DEBUG("using nfs share: %s\n", mod_uri); 670 use_nfs = 1; 671 err = load_mod_list_vfs(mod_list, 1, &modules, &modules_size); 672 } else if (strncmp(mod_uri, "tftp://", 7) == 0) { 673 use_tftp = 1; 674 char *del = strchr(mod_uri+7, ':');\ 675 uint16_t port = 69; // default tftp port 676 if (del != NULL) { 677 port = atoi(del + 1); 678 *del = 0; 679 } 680 681 XBOOT_DEBUG("using tftp server: %s @ port %u\n", mod_uri + 7, port); 682 683 err = tftp_client_connect(mod_uri + 7, port); 684 if (err_is_fail(err)) { 685 USER_PANIC_ERR(err, "Could not connect to the tftp service"); 686 } 687 err = load_mod_list_tftp(mod_list, &modules, &modules_size); 688 } else { 689 err = load_mod_list_vfs(mod_list, 0, &modules, &modules_size); 690 } 691 692 if (err_is_fail(err)) { 693 USER_PANIC_ERR(err, "failed to load modules list"); 694 } 695 696 char **modules_parsed = NULL; 697 uint32_t num_mods = 0, num_mmaps = 0; 698 uint8_t has_kernel = 0; 699 err = parse_mod_list(modules, &num_mods, &num_mmaps, &has_kernel, &modules_parsed); 700 if (err_is_fail(err)) { 701 return err; 702 } 703 704 if (!has_kernel) { 705 return SPAWN_ERR_FIND_MODULE; 706 } 707 708 // load the coprocessor OS (boot loader) 709 if (use_tftp) { 710 err = download_bootloader_tftp(phi, modules_parsed[0]); 711 } else { 712 err = download_bootloader_vfs(phi, modules_parsed[0], use_nfs); 713 } 714 if (err_is_fail(err)) { 715 USER_PANIC_ERR(err, "Could not load bootloader image"); 716 } 717 718 // round to next page 719 offset = ALIGN_BASE_PAGE(phi->os_offset + phi->os_size); 720 721 // load cmdline 722 err = load_cmdline(phi, offset); 723 if (err_is_fail(err)) { 724 USER_PANIC_ERR(err, "Could not load multiboot image"); 725 } 726 727 // round to next page 728 offset = ALIGN_BASE_PAGE(offset+phi->cmdlen); 729 730 if (use_tftp) { 731 err = download_modules_tftp(phi, offset, modules_parsed + 1, num_mods, num_mmaps); 732 } else { 733 err = download_modules_vfs(phi, offset, modules_parsed + 1, num_mods, num_mmaps, use_nfs); 734 } 735 if (err_is_fail(err)) { 736 USER_PANIC_ERR(err, "Could not load multiboot image"); 737 } 738 739 free(modules_parsed); 740 741 if (use_tftp) { 742 tftp_client_disconnect(); 743 } 744 745 struct xeon_phi_boot_params *bp; 746 bp = (struct xeon_phi_boot_params *)(phi->apt.vbase + phi->os_offset); 747 bp->xeon_phi_id = 0xFF00; 748 bp->xeon_phi_id += phi->id; 749 750 err = interphi_init(phi, NULL_CAP); 751 if (err_is_fail(err)) { 752 USER_PANIC_ERR(err, "Could not initialize messaging"); 753 } 754 755 xeon_phi_boot_download_status_wrf(&boot_registers, 0x0); 756 757 phi->apicid = xeon_phi_boot_download_apicid_rdf(&boot_registers); 758 759 // notify the bootstrap 760 bootstrap_notify(phi); 761 762 xeon_phi_boot_postcode_t postcode; 763 xeon_phi_boot_postcodes_t pc, pc_prev = 0; 764 uint32_t counter = BOOT_COUNTER; 765 while (--counter) { 766 postcode = xeon_phi_boot_postcode_rd(&boot_registers); 767 pc = xeon_phi_boot_postcode_code_extract(postcode); 768 if (pc_prev != pc) { 769 debug_printf("Xeon Phi Booting: %s\n", 770 xeon_phi_boot_postcodes_describe(pc)); 771 } 772 if (postcode == xeon_phi_boot_postcode_done) { 773 break; 774 } 775 pc_prev = pc; 776 } 777 778 XBOOT_DEBUG("Bootstrap has finished execution. Waiting for Firmware...\n"); 779 780 uint32_t time = 0, time_steps = 0; 781 while (time < BOOT_TIMEOUT) { 782 /* read all the pending messages */ 783 xeon_phi_serial_handle_recv(); 784 milli_sleep(100); 785 if (xeon_phi_boot_download_status_rdf(&boot_registers)) { 786 XBOOT_DEBUG("Firmware signaled with ready bit. \n"); 787 break; 788 } 789 if (!(time % 50)) { 790 debug_printf("Xeon Phi Booting: Waiting for ready signal %u\n", 791 time_steps); 792 time_steps += 5; 793 } 794 time++; 795 } 796 797 if (!xeon_phi_boot_download_status_rdf(&boot_registers)) { 798 USER_PANIC("Firmware not responding with ready bit"); 799 // TODO return error code 800 } 801 802 // we don't need the aperture mapped anymore so unmap it 803 err = xeon_phi_unmap_aperture(phi); 804 if (err_is_fail(err)) { 805 USER_PANIC_ERR(err, "Failed to map aperture range"); 806 } 807 808 return SYS_ERR_OK; 809} 810