1/** 2 * \file 3 * \brief Boot module for the Xeon Phi 4 * 5 * Loads the co processor OS onto the card and boots it 6 */ 7 8/* 9 * Copyright (c) 2014 ETH Zurich. 10 * All rights reserved. 11 * 12 * This file is distributed under the terms in the attached LICENSE file. 13 * If you do not find this file, copies can be found by writing to: 14 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 15 */ 16 17#include <stdio.h> 18#include <string.h> 19#include <barrelfish/barrelfish.h> 20#include <barrelfish/nameservice_client.h> 21#include <barrelfish/spawn_client.h> 22#include <skb/skb.h> 23 24#include <flounder/flounder_txqueue.h> 25 26#include <if/interphi_defs.h> 27 28#include <xeon_phi/xeon_phi.h> 29#include <xeon_phi/xeon_phi_domain.h> 30 31#include "xeon_phi_internal.h" 32#include "interphi.h" 33#include "smpt.h" 34#include "domain.h" 35#include "service.h" 36#include "xphi_service.h" 37#include "sysmem_caps.h" 38 39 40#include <driverkit/driverkit.h> 41#include <driverkit/iommu.h> 42#include <driverkit/hwmodel.h> 43#include "xeon_phi_internal.h" 44 45#define SIZE16G (16ll*1024*1024*1024) 46 47 48errval_t xeon_phi_hw_model_lookup_nodeids(int32_t pci_nodeid, int32_t *knc_socket, 49 int32_t *smpt, int32_t *iommu, 50 int32_t *dma, int32_t *k1om_core, 51 int32_t *gddr_node) 52{ 53 static int enums[6]; 54 static int skb_read = 0; 55 errval_t err; 56 57 if(!skb_read){ 58 skb_execute_query("decoding_net_listing"); 59 60 HWMODEL_QUERY_DEBUG("state_get(S),xeon_phi_meta_wrap(S, %"PRIi32").", 61 pci_nodeid); 62 err = skb_execute_query("state_get(S),xeon_phi_meta_wrap(S, %"PRIi32").", 63 pci_nodeid); 64 if (err_is_fail(err)) { 65 DEBUG_SKB_ERR(err, "query xeon phi meta. Retrying...."); 66 skb_execute_query("decoding_net_listing"); 67 68 HWMODEL_QUERY_DEBUG("state_get(S),xeon_phi_meta_wrap(S, %"PRIi32").", 69 pci_nodeid); 70 err = skb_execute_query("state_get(S),xeon_phi_meta_wrap(S, %"PRIi32").", 71 pci_nodeid); 72 if (err_is_fail(err)) { 73 DEBUG_SKB_ERR(err, "query xeon phi meta"); 74 } 75 return err; 76 } 77 err = skb_read_output("%d %d %d %d %d %d", 78 &enums[0], &enums[1], &enums[2], &enums[3], &enums[4], &enums[5]); 79 if(err_is_fail(err)) return err; 80 skb_read = true; 81 debug_printf("PHI NodeIds: KNC_SOCKET=%d, SMPT=%d, IOMMU=%d, DMA=%d," 82 " K1OM_CORE=%d\n", 83 enums[0], enums[1], enums[2], enums[3], enums[4]); 84 }; 85 86 if(knc_socket) *knc_socket = enums[0]; 87 if(smpt) *smpt = enums[1]; 88 if(iommu) *iommu = enums[2]; 89 if(dma) *dma = enums[3]; 90 if(k1om_core) *k1om_core = enums[4]; 91 if(gddr_node) *gddr_node = enums[5]; 92 93 return SYS_ERR_OK; 94} 95 96// Can install SMPT and IOMMU configs 97static errval_t install_config(struct xeon_phi *xphi, char * conf, struct dmem *dmem){ 98 errval_t err; 99 assert(!(skb_get_output() <= conf && conf <= skb_get_output() + 512)); 100 101 // Get configurable nodeids 102 int32_t smpt_id, iommu_id; 103 err = xeon_phi_hw_model_lookup_nodeids(xphi->nodeid, NULL, &smpt_id, &iommu_id, NULL, NULL, NULL); 104 if(err_is_fail(err)) { 105 DEBUG_ERR(err, "lookup nodeids"); 106 return err; 107 } 108 109 debug_printf("xphi nodeid: %d, iommu nodeid: %d, smpt nodeid: %d\n", 110 xphi->nodeid, iommu_id, smpt_id); 111 112 struct list_parser_status status; 113 skb_read_list_init_offset(&status, conf, 0); 114 uint64_t inaddr, outaddr; 115 int32_t nodeid; 116 dmem->devaddr = 0xfffffffffffffff; 117 bool iommu_needs_conf = false; 118 while(skb_read_list(&status, "c(%"SCNi32", %"SCNu64", %"SCNu64")", 119 &nodeid, &inaddr, &outaddr)) { 120 debug_printf("nodeid=%"PRIi32"\n", nodeid); 121 if(nodeid == smpt_id){ 122 debug_printf("CONF SMPT: in=0x%"PRIx64", out=0x%"PRIx64"\n", inaddr, outaddr); 123 smpt_set_address(xphi, inaddr / SIZE16G, outaddr, 1); 124 } else if(nodeid == iommu_id){ 125 debug_printf("CONF IOMMU: in=0x%"PRIx64", out=0x%"PRIx64"\n", inaddr, outaddr); 126 if(inaddr < dmem->devaddr){ 127 dmem->devaddr = inaddr; 128 iommu_needs_conf = true; 129 } 130 } else { 131 debug_printf("%s:%d: Don't know how to config node %d. Ignoring.\n", 132 __FUNCTION__, __LINE__, nodeid); 133 } 134 } 135 136 dmem->vbase = 0; 137 if(iommu_needs_conf){ 138 // Fills in dmem->vbase 139 err = driverkit_iommu_vspace_map_fixed_cl(xphi->iommu_client, dmem->mem, 140 VREGION_FLAGS_READ_WRITE, dmem); 141 if(err_is_fail(err)){ 142 DEBUG_ERR(err, "map_fixed"); 143 } 144 return err; 145 } else { 146 return SYS_ERR_OK; 147 } 148} 149 150/* 151 * the translate address is called when a region is registered 152 */ 153errval_t xeon_phi_hw_model_query_and_config(void *arg, 154 struct capref mem, 155 genpaddr_t *retaddr, 156 genvaddr_t *local_retaddr) 157{ 158 errval_t err; 159 struct xeon_phi *xphi = arg; 160 assert(arg != NULL); 161 assert(retaddr != NULL); 162 163 struct frame_identity id; 164 err = frame_identify(mem, &id); 165 if (err_is_fail(err)) { 166 return err; 167 } 168 169 /* query the model */ 170 #ifdef __k1om__ 171 return LIB_ERR_NOT_IMPLEMENTED; 172 #endif 173 174 char conf[1024]; 175 int32_t knc_sock_id; 176 err = xeon_phi_hw_model_lookup_nodeids(xphi->nodeid, &knc_sock_id, NULL, NULL, NULL, NULL, NULL); 177 if(err_is_fail(err)) { 178 DEBUG_ERR(err, "lookup nodeids"); 179 return err; 180 } 181 182 err = driverkit_hwmodel_get_map_conf(mem, knc_sock_id, conf, sizeof(conf), retaddr); 183 assert(err_is_ok(err)); 184 185 skb_execute_query("decoding_net_listing"); 186 187 debug_printf("[knc] Translated address into Xeon Phi space: 0x%"PRIx64"\n", 188 *retaddr); 189 190 191 struct dmem dmem; 192 dmem.size = id.bytes; 193 dmem.mem = mem; 194 err = install_config(xphi, conf, &dmem); 195 assert(err_is_ok(err)); 196 197 /* TODO: only do this if IOMMU is present or needs to be changed */ 198 199 if (err_is_fail(err)) { 200 DEBUG_ERR(err, "TODO: CLEANUP!"); 201 } 202 203 if(local_retaddr) { 204 // This is a bit unfortunate. We only generate dmem.vbase 205 // if we setup the IOMMU. But since most caller dont care this works. 206 assert(dmem.vbase); 207 *local_retaddr = dmem.vbase; 208 } 209 210 return SYS_ERR_OK; 211} 212