1/* 2 * Copyright (c) 2014, ETH Zurich. All rights reserved. 3 * 4 * This file is distributed under the terms in the attached LICENSE file. 5 * If you do not find this file, copies can be found by writing to: 6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 7 */ 8 9#include <string.h> 10#include <barrelfish/barrelfish.h> 11#include <bench/bench.h> 12#include <driverkit/iommu.h> 13#include <driverkit/hwmodel.h> 14#include <dev/xeon_phi/xeon_phi_dma_dev.h> 15 16#include <dma_mem_utils.h> 17 18#include <xeon_phi/xeon_phi_dma_internal.h> 19#include <xeon_phi/xeon_phi_dma_device_internal.h> 20#include <xeon_phi/xeon_phi_dma_channel_internal.h> 21 22#include <debug.h> 23 24/** 25 * Xeon Phi DMA device representation 26 */ 27struct xeon_phi_dma_device 28{ 29 struct dma_device common; 30 struct iommu_client *iommu; 31 xeon_phi_dma_t device; ///< mackerel device base 32 struct dmem dstat; ///< memory region for channels dstat_wb 33 uint32_t flags; 34}; 35 36/// counter for device ID enumeration 37static dma_dev_id_t device_id = 1; 38 39/* 40 * =========================================================================== 41 * Library Internal Interface 42 * =========================================================================== 43 */ 44 45/** 46 * \brief fills in the memory information structure for the channel's dstat 47 * address 48 * 49 * \param dev Xeon Phi DMA device 50 * \param mem Memory structure to fill in 51 */ 52void xeon_phi_dma_device_get_dstat_addr(struct xeon_phi_dma_device *dev, 53 struct dmem *mem) 54{ 55 assert(dev->dstat.vbase); 56 57 *mem = dev->dstat; 58 mem->size = XEON_PHI_DMA_CHANNEL_DSTAT_SIZE; 59 mem->devaddr += (XEON_PHI_DMA_CHANNEL_DSTAT_SIZE * dev->common.channels.next); 60 mem->mem = NULL_CAP; 61 mem->vbase += (XEON_PHI_DMA_CHANNEL_DSTAT_SIZE * dev->common.channels.next++); 62} 63 64/** 65 * \brief globally enables the interrupts for the given device 66 * 67 * \param dev Xeon Phi DMA device 68 * \param type the interrupt type to enable 69 */ 70errval_t xeon_phi_dma_device_irq_setup(struct xeon_phi_dma_device *dev, 71 dma_irq_t type) 72{ 73 assert(!"NYI"); 74 75 return SYS_ERR_OK; 76} 77 78/** 79 * \brief gets the Xeon Phi virtual base address of the DMA channel with 80 * the given id 81 * 82 * \param dev Xeon Phi DMA device 83 * \param idx DMA channel index 84 * 85 * \returns virtual address of MMIO registers for the channel 86 */ 87void *xeon_phi_dma_device_get_channel_vbase(struct xeon_phi_dma_device *dev, 88 uint8_t idx) 89{ 90 XPHIDEV_DEBUG("getting channel vbase for %u, offset=%x\n", dev->common.id, 91 idx, (idx * 0x40) + XEON_PHI_DMA_OFFSET); 92 return (void *) (dev->common.mmio.vaddr + (idx * 0x40) + XEON_PHI_DMA_OFFSET); 93} 94 95/** 96 * \brief sets the channel owner register of the Xeon Phi DMA device 97 * 98 * \param dev Xeon Phi DMA device 99 * \param idx channel index 100 * \param owner owner of the channel 101 */ 102void xeon_phi_dma_device_set_channel_owner(struct xeon_phi_dma_device *dev, 103 uint8_t idx, 104 xeon_phi_dma_owner_t owner) 105{ 106 uint8_t owner_val; 107 if (owner == XEON_PHI_DMA_OWNER_CARD) { 108 XPHIDEV_DEBUG("settings owner of channel [%u] to card.\n", dev->common.id, idx); 109 owner_val = 0; 110 } else { 111 XPHIDEV_DEBUG("settings owner of channel [%u] to host.\n", dev->common.id, idx); 112 owner_val = 1; 113 } 114 115 xeon_phi_dma_dcr_t dcr = xeon_phi_dma_dcr_rd(&dev->device); 116 117 switch (idx) { 118 case 0x0: 119 dcr = xeon_phi_dma_dcr_co0_insert(dcr, owner_val); 120 break; 121 case 0x1: 122 dcr = xeon_phi_dma_dcr_co1_insert(dcr, owner_val); 123 break; 124 case 0x2: 125 dcr = xeon_phi_dma_dcr_co2_insert(dcr, owner_val); 126 break; 127 case 0x3: 128 dcr = xeon_phi_dma_dcr_co3_insert(dcr, owner_val); 129 break; 130 case 0x4: 131 dcr = xeon_phi_dma_dcr_co4_insert(dcr, owner_val); 132 break; 133 case 0x5: 134 dcr = xeon_phi_dma_dcr_co5_insert(dcr, owner_val); 135 break; 136 case 0x6: 137 dcr = xeon_phi_dma_dcr_co6_insert(dcr, owner_val); 138 break; 139 case 0x7: 140 dcr = xeon_phi_dma_dcr_co7_insert(dcr, owner_val); 141 break; 142 } 143 144 xeon_phi_dma_dcr_wr(&dev->device, dcr); 145} 146 147/** 148 * \brief Enables / Disables the Xeon Phi DMA channel 149 * 150 * \param chan Xeon Phi DMA channel 151 * \param idx channel index 152 * \param enabled flag to set the channel enabled 153 */ 154void xeon_phi_dma_device_set_channel_state(struct xeon_phi_dma_device *dev, 155 uint8_t idx, 156 uint8_t enabled) 157{ 158 uint8_t id = idx - XEON_PHI_DMA_DEVICE_CHAN_OFFSET; 159 160 xeon_phi_dma_dcr_t dcr = xeon_phi_dma_dcr_rd(&dev->device); 161 162 uint8_t enabled_val; 163 if (enabled) { 164 XPHIDEV_DEBUG("Enabling channel. [%u]\n", dev->common.id, idx); 165 if (dev->common.channels.c[id]) { 166 dev->common.channels.c[id]->state = DMA_CHAN_ST_RUNNING; 167 } 168 enabled_val = 0x1; 169 } else { 170 XPHIDEV_DEBUG("Disabling channel. [%u]\n", dev->common.id, idx); 171 if (dev->common.channels.c[id]) { 172 dev->common.channels.c[id]->state = DMA_CHAN_ST_SUSPENDED; 173 } 174 enabled_val = 0x0; 175 } 176 177 switch (idx) { 178 case 0x0: 179 dcr = xeon_phi_dma_dcr_ce0_insert(dcr, enabled_val); 180 break; 181 case 0x1: 182 dcr = xeon_phi_dma_dcr_ce1_insert(dcr, enabled_val); 183 break; 184 case 0x2: 185 dcr = xeon_phi_dma_dcr_ce2_insert(dcr, enabled_val); 186 break; 187 case 0x3: 188 dcr = xeon_phi_dma_dcr_ce3_insert(dcr, enabled_val); 189 break; 190 case 0x4: 191 dcr = xeon_phi_dma_dcr_ce4_insert(dcr, enabled_val); 192 break; 193 case 0x5: 194 dcr = xeon_phi_dma_dcr_ce5_insert(dcr, enabled_val); 195 break; 196 case 0x6: 197 dcr = xeon_phi_dma_dcr_ce6_insert(dcr, enabled_val); 198 break; 199 case 0x7: 200 dcr = xeon_phi_dma_dcr_ce7_insert(dcr, enabled_val); 201 break; 202 } 203 204 xeon_phi_dma_dcr_wr(&dev->device, dcr); 205} 206 207/* 208 * =========================================================================== 209 * Public Interface 210 * =========================================================================== 211 */ 212 213/* 214 * ---------------------------------------------------------------------------- 215 * device initialization / termination 216 * ---------------------------------------------------------------------------- 217 */ 218 219#include <xeon_phi/xeon_phi.h> 220 221/** 222 * \brief initializes a Xeon Phi DMA device with the giving capability 223 * 224 * \param mmio capability representing the device's MMIO registers 225 * \param iommu 226 * \param convert memory conversion function. In a simple case, returns the 227 * physical address of the cap. 228 * \param nodeid HW model node id where xphi requests originate from, necessary 229 * for model supported frame allocation. 230 * \param dev returns a pointer to the device structure 231 * 232 * 233 * \returns SYS_ERR_OK on success 234 * errval on error 235 */ 236errval_t xeon_phi_dma_device_init(void *mmio_base, struct iommu_client *iommu, 237 dma_mem_convert_fn convert, void *convert_arg, 238 int32_t nodeid, 239 struct xeon_phi_dma_device **dev) 240{ 241 errval_t err; 242 243 struct xeon_phi_dma_device *xdev = calloc(1, sizeof(*xdev)); 244 if (xdev == NULL) { 245 return LIB_ERR_MALLOC_FAIL; 246 } 247 248#if DMA_BENCH_ENABLED 249 bench_init(); 250#endif 251 252 struct dma_device *dma_dev = (struct dma_device *) xdev; 253 254 XPHIDEV_DEBUG("initializing Xeon Phi DMA device @ %p\n", device_id, 255 mmio_base); 256 257 258#if defined(XEON_PHI_USE_HW_MODEL) 259 int32_t nodes[3]; 260 nodes[0] = nodeid; 261 nodes[1] = driverkit_hwmodel_get_my_node_id(); 262 nodes[2] = 0; 263 int32_t dest_nodeid = driverkit_hwmodel_lookup_dram_node_id(); 264 265 err = driverkit_hwmodel_frame_alloc(&xdev->dstat.mem, LARGE_PAGE_SIZE, 266 dest_nodeid, nodes); 267 if (err_is_fail(err)) { 268 free(xdev); 269 return err; 270 } 271 272 err = convert(convert_arg, xdev->dstat.mem, &xdev->dstat.devaddr, &xdev->dstat.vbase); 273 if (err_is_fail(err)) { 274 DEBUG_ERR(err, "convert mem"); 275 free(xdev); 276 return err; 277 } 278#else 279 // Alloc cap slota 280 err = dma_mem_alloc(LARGE_PAGE_SIZE, VREGION_FLAGS_READ_WRITE, NULL, &xdev->dstat); 281 if (err_is_fail(err)) { 282 free(xdev); 283 return err; 284 } 285#endif 286 287 //debug_printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx\n"); 288 //debug_printf("HACK: setting address!\n"); 289 //xdev->dstat.devaddr = xdev->dstat.devaddr + XEON_PHI_SYSMEM_BASE - 2 * (512UL << 30); 290 //debug_printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx\n");R 291 292 dma_dev->id = device_id++; 293 dma_dev->irq_type = DMA_IRQ_DISABLED; 294 dma_dev->type = DMA_DEV_TYPE_XEON_PHI; 295 dma_dev->mmio.vaddr = (lvaddr_t) mmio_base; 296 dma_dev->f.poll = xeon_phi_dma_device_poll_channels; 297 dma_dev->iommu = iommu; 298 dma_dev->convert = convert; 299 dma_dev->convert_arg = convert_arg; 300 dma_dev->nodeid = nodeid; 301 302 xeon_phi_dma_initialize(&xdev->device, mmio_base); 303 304 XPHIDEV_DEBUG("initializing %u channels\n", device_id, 305 XEON_PHI_DMA_DEVICE_CHANNELS); 306 307 dma_dev->channels.count = XEON_PHI_DMA_DEVICE_CHANNELS; 308 dma_dev->channels.c = calloc(XEON_PHI_DMA_DEVICE_CHANNELS, 309 sizeof(void *)); 310 311 if (dma_dev->channels.c == NULL) { 312 device_id--; 313 free(xdev); 314 return LIB_ERR_MALLOC_FAIL; 315 } 316 317 for (uint8_t i = 0; i < XEON_PHI_DMA_DEVICE_CHANNELS; ++i) { 318 struct dma_channel **chan = &dma_dev->channels.c[i]; 319 err = xeon_phi_dma_channel_init(xdev, i, XEON_PHI_DMA_DEVICE_MAX_XFER, 320 (struct xeon_phi_dma_channel **) chan); 321 if (err_is_fail(err)) { 322 free(dma_dev->channels.c); 323 device_id--; 324 free(xdev); 325 return err; 326 } 327 } 328 329 *dev = xdev; 330 331 XPHIDEV_DEBUG("Xeon Phi DMA device initialized\n", dma_dev->id); 332 333 return err; 334} 335 336/** 337 * \brief terminates the device operation and frees up the allocated resources 338 * 339 * \param dev Xeon Phi DMA device to shutdown 340 * 341 * \returns SYS_ERR_OK on success 342 * errval on error 343 */ 344errval_t xeon_phi_dma_device_shutdown(struct xeon_phi_dma_device *dev) 345{ 346 assert(!"NYI"); 347 return SYS_ERR_OK; 348} 349 350/* 351 * ---------------------------------------------------------------------------- 352 * Interrupt management 353 * ---------------------------------------------------------------------------- 354 */ 355 356/** 357 * \brief enables the interrupts for the device 358 * 359 * \param dev Xeon Phi DMA device 360 * \param type interrupt type 361 * \param fn interrupt handler function 362 * \param arg argument supplied to the handler function 363 */ 364errval_t xeon_phi_dma_device_intr_enable(struct xeon_phi_dma_device *dev, 365 dma_irq_t type, 366 dma_irq_fn_t fn, 367 void *arg) 368{ 369 assert(!"NYI"); 370 return SYS_ERR_OK; 371} 372 373/** 374 * \brief disables the interrupts for the device 375 * 376 * \param dev Xeon Phi DMA device 377 */ 378void xeon_phi_dma_device_intr_disable(struct xeon_phi_dma_device *dev) 379{ 380 assert(!"NYI"); 381} 382 383/* 384 * ---------------------------------------------------------------------------- 385 * Device Operation Functions 386 * ---------------------------------------------------------------------------- 387 */ 388 389/** 390 * \brief polls the channels of the Xeon Phi DMA device 391 * 392 * \param dev Xeon Phi DMA device 393 * 394 * \returns SYS_ERR_OK on success 395 * DMA_ERR_DEVICE_IDLE if there is nothing completed on the channels 396 * errval on error 397 */ 398errval_t xeon_phi_dma_device_poll_channels(struct dma_device *dev) 399{ 400 errval_t err; 401 402 uint8_t idle = 0x1; 403 404 for (uint8_t i = 0; i < dev->channels.count; ++i) { 405 err = xeon_phi_dma_channel_poll(dev->channels.c[i]); 406 switch (err_no(err)) { 407 case DMA_ERR_CHAN_IDLE: 408 /* no op */ 409 break; 410 case SYS_ERR_OK: 411 idle = 0; 412 break; 413 default: 414 return err; 415 } 416 } 417 418 if (idle) { 419 return DMA_ERR_DEVICE_IDLE; 420 } 421 422 return SYS_ERR_OK; 423} 424 425