1/* 2 * Copyright (c) 2014, ETH Zurich. All rights reserved. 3 * 4 * This file is distributed under the terms in the attached LICENSE file. 5 * If you do not find this file, copies can be found by writing to: 6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group. 7 */ 8#include <string.h> 9#include <barrelfish/barrelfish.h> 10#include <barrelfish/waitset.h> 11#include <bench/bench.h> 12#include <dev/ioat_dma_dev.h> 13 14#include <dma_mem_utils.h> 15 16#include <ioat/ioat_dma_internal.h> 17#include <ioat/ioat_dma_dca_internal.h> 18#include <ioat/ioat_dma_device_internal.h> 19#include <ioat/ioat_dma_channel_internal.h> 20 21#include <debug.h> 22 23/** 24 * IOAT DMA device representation 25 */ 26struct ioat_dma_device 27{ 28 struct dma_device common; 29 30 ioat_dma_t device; ///< mackerel device base 31 ioat_dma_cbver_t version; ///< Crystal Beach version number 32 33 struct dma_mem complstatus; ///< memory region for channels CHANSTS 34 struct pci_addr pci_addr; ///< the PCI address of the device 35 36 uint8_t irq_msix_vector; 37 uint16_t irq_msix_count; 38 39 uint32_t flags; 40}; 41 42/// counter for device ID enumeration 43static dma_dev_id_t device_id = 1; 44 45/* 46 * ---------------------------------------------------------------------------- 47 * device initialization functions 48 * ---------------------------------------------------------------------------- 49 */ 50 51static errval_t device_init_ioat_v1(struct ioat_dma_device *dev) 52{ 53 IOATDEV_DEBUG("devices of Crystal Beach Version 1.xx are currently not supported.\n", 54 dev->common.id); 55 return DMA_ERR_DEVICE_UNSUPPORTED; 56} 57 58static errval_t device_init_ioat_v2(struct ioat_dma_device *dev) 59{ 60 IOATDEV_DEBUG("devices of Crystal Beach Version 2.xx are currently not supported.\n", 61 dev->common.id); 62 return DMA_ERR_DEVICE_UNSUPPORTED; 63} 64 65static errval_t device_init_ioat_v3(struct ioat_dma_device *dev) 66{ 67 errval_t err; 68 69 IOATDEV_DEBUG("initialize Crystal Beach 3 DMA device\n", dev->common.id); 70 71 ioat_dma_dmacapability_t cap = ioat_dma_dmacapability_rd(&dev->device); 72 73 if (ioat_dma_cbver_minor_extract(dev->version) == 2) { 74 IOATDEV_DEBUG("disabling XOR and PQ opcodes for Crystal Beach 3.2\n", 75 dev->common.id); 76 cap = ioat_dma_dmacapability_xor_insert(cap, 0x0); 77 cap = ioat_dma_dmacapability_pq_insert(cap, 0x0); 78 } else if (ioat_dma_cbver_minor_extract(dev->version) == 3) { 79 IOATDEV_DEBUG("devices of Crystal Beach Version 3.3 are not supported.\n", 80 dev->common.id); 81 return DMA_ERR_DEVICE_UNSUPPORTED; 82 } 83 84 /* if DCA is enabled, we cannot support the RAID functions */ 85 if (ioat_dma_dca_is_enabled()) { 86 IOATDEV_DEBUG("Disabling XOR and PQ while DCA is enabled\n", dev->common.id); 87 cap = ioat_dma_dmacapability_xor_insert(cap, 0x0); 88 cap = ioat_dma_dmacapability_pq_insert(cap, 0x0); 89 } 90 91 if (ioat_dma_dmacapability_xor_extract(cap)) { 92 IOATDEV_DEBUG("device supports XOR RAID.\n", dev->common.id); 93 94 dev->flags |= IOAT_DMA_DEV_F_RAID; 95 96 /* 97 * this may need some additional functions to prepare 98 * the specific transfers... 99 * 100 * max_xor = 8; 101 * prepare_xor, prepare_xor_val 102 */ 103 } 104 105 if (ioat_dma_dmacapability_pq_extract(cap)) { 106 IOATDEV_DEBUG("device supports PQ RAID.\n", dev->common.id); 107 108 dev->flags |= IOAT_DMA_DEV_F_RAID; 109 110 /* 111 * this may need some additional functions to prepare the 112 * DMA descriptors 113 * 114 * max_xor = 8; 115 * max_pq = 8; 116 * prepare_pq, perpare_pq_val 117 * 118 * also set the prepare_xor pointers... 119 * 120 */ 121 } 122 123 /* set the interrupt type to disabled*/ 124 dev->common.irq_type = DMA_IRQ_DISABLED; 125 dev->common.type = DMA_DEV_TYPE_IOAT; 126 127 /* allocate memory for completion status writeback */ 128 err = dma_mem_alloc(IOAT_DMA_COMPLSTATUS_SIZE, IOAT_DMA_COMPLSTATUS_FLAGS, 129 &dev->complstatus); 130 if (err_is_fail(err)) { 131 return err; 132 } 133 134 dev->common.channels.count = ioat_dma_chancnt_num_rdf(&dev->device); 135 136 dev->common.channels.c = calloc(dev->common.channels.count, 137 sizeof(*dev->common.channels.c)); 138 if (dev->common.channels.c == NULL) { 139 dma_mem_free(&dev->complstatus); 140 return LIB_ERR_MALLOC_FAIL; 141 } 142 143 /* channel enumeration */ 144 145 IOATDEV_DEBUG("channel enumeration. discovered %u channels\n", dev->common.id, 146 dev->common.channels.count); 147 148 uint32_t max_xfer_size = (1 << ioat_dma_xfercap_max_rdf(&dev->device)); 149 150 for (uint8_t i = 0; i < dev->common.channels.count; ++i) { 151 struct dma_channel **chan = &dev->common.channels.c[i]; 152 err = ioat_dma_channel_init(dev, i, max_xfer_size, 153 (struct ioat_dma_channel **) chan); 154 } 155 156 if (dev->flags & IOAT_DMA_DEV_F_DCA) { 157 /* 158 * TODO: DCA initialization 159 * device->dca = ioat3_dca_init(pdev, device->reg_base); 160 */ 161 } 162 163 err = ioat_dma_device_irq_setup(dev, DMA_IRQ_MSIX); 164 if (err_is_fail(err)) { 165 return err; 166 } 167 168 return SYS_ERR_OK; 169} 170 171/* 172 * =========================================================================== 173 * Library Internal Interface 174 * =========================================================================== 175 */ 176 177void ioat_dma_device_get_complsts_addr(struct ioat_dma_device *dev, 178 struct dma_mem *mem) 179{ 180 if (dev->common.state != DMA_DEV_ST_CHAN_ENUM) { 181 memset(mem, 0, sizeof(*mem)); 182 } 183 184 assert(dev->complstatus.vaddr); 185 186 *mem = dev->complstatus; 187 mem->bytes = IOAT_DMA_COMPLSTATUS_SIZE; 188 mem->paddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next); 189 mem->frame = NULL_CAP; 190 mem->vaddr += (IOAT_DMA_COMPLSTATUS_SIZE * dev->common.channels.next++); 191} 192 193#if IOAT_DEBUG_INTR_ENABLED 194///< flag indicating that the interrupt has happened for debugging purposes 195static uint32_t msix_intr_happened = 0; 196#include <dma/ioat/ioat_dma_request.h> 197#endif 198 199static void ioat_dma_device_irq_handler(void* arg) 200{ 201 errval_t err; 202 struct dma_device *dev = arg; 203 204 IOATDEV_DEBUG("############ MSIX INTERRUPT HAPPENED.\n", dev->id); 205 206#if IOAT_DEBUG_INTR_ENABLED 207 msix_intr_happened=1; 208#endif 209 210 err = ioat_dma_device_poll_channels(dev); 211 if (err_is_fail(err)) { 212 if (err_no(err) == DMA_ERR_DEVICE_IDLE) { 213 IOATDEV_DEBUG("WARNING: MSI-X interrupt on idle device\n", dev->id); 214 return; 215 } 216 USER_PANIC_ERR(err, "dma poll device returned an error\n"); 217 } 218} 219 220/** 221 * \brief gets the local apic ID from the CPU id 222 * 223 * \return local apic ID 224 */ 225static inline uint8_t get_local_apic_id(void) 226{ 227 uint32_t eax, ebx; 228 229 cpuid(1, &eax, &ebx, NULL, NULL); 230 return ebx >> 24; 231} 232 233/** 234 * \brief globally enables the interrupts for the given device 235 * 236 * \param dev IOAT DMA device 237 * \param type the interrupt type to enable 238 */ 239errval_t ioat_dma_device_irq_setup(struct ioat_dma_device *dev, 240 dma_irq_t type) 241{ 242 errval_t err; 243 244 ioat_dma_intrctrl_t intcrtl = 0; 245 intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1); 246 247 dev->common.irq_type = type; 248 switch (type) { 249 case DMA_IRQ_MSIX: 250 /* The number of MSI-X vectors should equal the number of channels */ 251 IOATDEV_DEBUG("MSI-X interrupt setup for device (%u, %u, %u)\n", 252 dev->common.id, dev->pci_addr.bus, dev->pci_addr.device, 253 dev->pci_addr.function); 254 255 err = pci_msix_enable_addr(&dev->pci_addr, &dev->irq_msix_count); 256 if (err_is_fail(err)) { 257 return err; 258 } 259 260 assert(dev->irq_msix_count > 0); 261 262 IOATDEV_DEBUG("MSI-X enabled #vecs=%d\n", dev->common.id, 263 dev->irq_msix_count); 264 265 err = pci_setup_inthandler(ioat_dma_device_irq_handler, dev, 266 &dev->irq_msix_vector); 267 assert(err_is_ok(err)); 268 269 uint8_t dest = get_local_apic_id(); 270 271 IOATDEV_DEBUG("MSI-X routing to apic=%u\n", dev->common.id, 272 dest); 273 274 err = pci_msix_vector_init_addr(&dev->pci_addr, 0, dest, 275 dev->irq_msix_vector); 276 assert(err_is_ok(err)); 277 278 /* enable the interrupts */ 279 intcrtl = ioat_dma_intrctrl_msix_vec_insert(intcrtl, 1); 280 intcrtl = ioat_dma_intrctrl_intp_en_insert(intcrtl, 1); 281 break; 282 case DMA_IRQ_MSI: 283 IOATDEV_DEBUG("Initializing MSI interrupts \n", dev->common.id); 284 assert(!"NYI"); 285 break; 286 case DMA_IRQ_INTX: 287 IOATDEV_DEBUG("Initializing INTx interrupts \n", dev->common.id); 288 assert(!"NYI"); 289 break; 290 default: 291 /* disabled */ 292 intcrtl = 0; 293 IOATDEV_DEBUG("Disabling interrupts \n", dev->common.id); 294 break; 295 } 296 297 ioat_dma_intrctrl_wr(&dev->device, intcrtl); 298 299 300#if IOAT_DEBUG_INTR_ENABLED 301 /* 302 * check if interrupts are working. 303 */ 304 msix_intr_happened = 0; 305 306 struct ioat_dma_channel *chan; 307 chan = (struct ioat_dma_channel *)dev->common.channels.c[0]; 308 309 ioat_dma_request_nop_chan(chan); 310 err = ioat_dma_channel_issue_pending(chan); 311 if (err_is_fail(err)) { 312 return err; 313 } 314 315 while(msix_intr_happened == 0) { 316 uint64_t status = ioat_dma_channel_get_status(chan); 317 err = event_dispatch_non_block(get_default_waitset()); 318 319 if (!ioat_dma_channel_is_active(status) && !ioat_dma_channel_is_idle(status)) { 320 USER_PANIC("DMA request turned channel into erroneous state.") 321 } 322 323 switch(err_no(err)) { 324 case LIB_ERR_NO_EVENT: 325 thread_yield(); 326 break; 327 case SYS_ERR_OK: 328 continue; 329 default: 330 USER_PANIC_ERR(err, "dispatching event"); 331 } 332 } 333#endif 334 335 return SYS_ERR_OK; 336} 337 338/* 339 * =========================================================================== 340 * Public Interface 341 * =========================================================================== 342 */ 343 344/* 345 * ---------------------------------------------------------------------------- 346 * device initialization / termination 347 * ---------------------------------------------------------------------------- 348 */ 349 350/** 351 * \brief initializes a IOAT DMA device with the giving capability 352 * 353 * \param mmio capability representing the device's MMIO registers 354 * \param pci_addr the PCI address of this device 355 * \param dev returns a pointer to the device structure 356 * 357 * \returns SYS_ERR_OK on success 358 * errval on error 359 */ 360errval_t ioat_dma_device_init(struct capref mmio, 361 struct pci_addr *pci_addr, 362 struct ioat_dma_device **dev) 363{ 364 errval_t err; 365 366 struct ioat_dma_device *ioat_device = calloc(1, sizeof(*ioat_device)); 367 if (ioat_device == NULL) { 368 return LIB_ERR_MALLOC_FAIL; 369 } 370 371#if DMA_BENCH_ENABLED 372 bench_init(); 373#endif 374 375 struct dma_device *dma_dev = &ioat_device->common; 376 377 struct frame_identity mmio_id; 378 err = frame_identify(mmio, &mmio_id); 379 if (err_is_fail(err)) { 380 free(ioat_device); 381 return err; 382 } 383 384 dma_dev->id = device_id++; 385 dma_dev->mmio.paddr = mmio_id.base; 386 dma_dev->mmio.bytes = mmio_id.bytes; 387 dma_dev->mmio.frame = mmio; 388 ioat_device->pci_addr = *pci_addr; 389 390 IOATDEV_DEBUG("init device with mmio range: {paddr=0x%016lx, size=%u kB}\n", 391 dma_dev->id, mmio_id.base, mmio_id.bytes / 1024); 392 393 err = vspace_map_one_frame_attr((void**) &dma_dev->mmio.vaddr, 394 dma_dev->mmio.bytes, dma_dev->mmio.frame, 395 VREGION_FLAGS_READ_WRITE_NOCACHE, 396 NULL, NULL); 397 if (err_is_fail(err)) { 398 free(ioat_device); 399 return err; 400 } 401 402 ioat_dma_initialize(&ioat_device->device, NULL, (void *) dma_dev->mmio.vaddr); 403 404 ioat_device->version = ioat_dma_cbver_rd(&ioat_device->device); 405 406 IOATDEV_DEBUG("device registers mapped at 0x%016lx. IOAT version: %u.%u\n", 407 dma_dev->id, dma_dev->mmio.vaddr, 408 ioat_dma_cbver_major_extract(ioat_device->version), 409 ioat_dma_cbver_minor_extract(ioat_device->version)); 410 411 switch (ioat_dma_cbver_major_extract(ioat_device->version)) { 412 case ioat_dma_cbver_1x: 413 err = device_init_ioat_v1(ioat_device); 414 break; 415 case ioat_dma_cbver_2x: 416 err = device_init_ioat_v2(ioat_device); 417 break; 418 case ioat_dma_cbver_3x: 419 err = device_init_ioat_v3(ioat_device); 420 break; 421 default: 422 err = DMA_ERR_DEVICE_UNSUPPORTED; 423 } 424 425 if (err_is_fail(err)) { 426 vspace_unmap((void*) dma_dev->mmio.vaddr); 427 free(ioat_device); 428 return err; 429 } 430 431 dma_dev->f.deregister_memory = NULL; 432 dma_dev->f.register_memory = NULL; 433 dma_dev->f.poll = ioat_dma_device_poll_channels; 434 435 *dev = ioat_device; 436 437 return err; 438} 439 440/** 441 * \brief terminates the device operation and frees up the allocated resources 442 * 443 * \param dev IOAT DMA device to shutdown 444 * 445 * \returns SYS_ERR_OK on success 446 * errval on error 447 */ 448errval_t ioat_dma_device_shutdown(struct ioat_dma_device *dev) 449{ 450 assert(!"NYI"); 451 return SYS_ERR_OK; 452} 453 454/** 455 * \brief requests access to a IOAT DMA device from the IOAT device manager 456 * and initializes the device. 457 * 458 * \param dev returns a pointer to the device structure 459 * 460 * \returns SYS_ERR_OK on success 461 * errval on error 462 */ 463errval_t ioat_dma_device_acquire(struct ioat_dma_device **dev) 464{ 465 errval_t err; 466 467 struct ioat_dma_device *ioat_device = calloc(1, sizeof(*ioat_device)); 468 if (ioat_device == NULL) { 469 return LIB_ERR_MALLOC_FAIL; 470 } 471 assert(!"NYI"); 472 err = SYS_ERR_OK; 473 return err; 474} 475 476/** 477 * \brief terminates the device operation and frees up the allocated resources 478 * and releases the device and returns it to the IOAT device manager. 479 * 480 * \param dev IOAT DMA device to be released 481 * 482 * \returns SYS_ERR_OK on success 483 * errval on error 484 */ 485errval_t ioat_dma_device_release(struct ioat_dma_device *dev) 486{ 487 assert(!"NYI"); 488 return SYS_ERR_OK; 489} 490 491/* 492 * ---------------------------------------------------------------------------- 493 * Interrupt management 494 * ---------------------------------------------------------------------------- 495 */ 496 497/** 498 * \brief enables the interrupts for the device 499 * 500 * \param dev IOAT DMA device 501 * \param type interrupt type 502 * \param fn interrupt handler function 503 * \param arg argument supplied to the handler function 504 */ 505errval_t ioat_dma_device_intr_enable(struct ioat_dma_device *dev, 506 dma_irq_t type, 507 dma_irq_fn_t fn, 508 void *arg) 509{ 510 assert(!"NYI"); 511 return SYS_ERR_OK; 512} 513 514/** 515 * \brief disables the interrupts for the device 516 * 517 * \param dev IOAT DMA device 518 */ 519void ioat_dma_device_intr_disable(struct ioat_dma_device *dev) 520{ 521 assert(!"NYI"); 522} 523 524/** 525 * \brief sets the interrupt delay for the device 526 * 527 * \param dev IOAT DMA device 528 * \param usec interrupt delay in microseconds 529 */ 530void ioat_dma_device_set_intr_delay(struct ioat_dma_device *dev, 531 uint16_t usec) 532{ 533 ioat_dma_intrdelay_delay_us_wrf(&dev->device, usec); 534} 535 536/* 537 * ---------------------------------------------------------------------------- 538 * Device Operation Functions 539 * ---------------------------------------------------------------------------- 540 */ 541 542/** 543 * \brief polls the channels of the IOAT DMA device 544 * 545 * \param dev IOAT DMA device 546 * 547 * \returns SYS_ERR_OK on success 548 * DMA_ERR_DEVICE_IDLE if there is nothing completed on the channels 549 * errval on error 550 */ 551errval_t ioat_dma_device_poll_channels(struct dma_device *dev) 552{ 553 errval_t err; 554 555 uint8_t idle = 0x1; 556 557 for (uint8_t i = 0; i < dev->channels.count; ++i) { 558 err = ioat_dma_channel_poll(dev->channels.c[i]); 559 switch (err_no(err)) { 560 case DMA_ERR_CHAN_IDLE: 561 break; 562 case SYS_ERR_OK: 563 idle = 0; 564 break; 565 default: 566 return err; 567 } 568 } 569 570 if (idle) { 571 return DMA_ERR_DEVICE_IDLE; 572 } 573 574 return SYS_ERR_OK; 575} 576 577