1/* 2 * Copyright (c) 2007-2011, ETH Zurich. 3 * All rights reserved. 4 * 5 * This file is distributed under the terms in the attached LICENSE file. 6 * If you do not find this file, copies can be found by writing to: 7 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group. 8 */ 9 10#include <barrelfish/barrelfish.h> 11#include <barrelfish/nameservice_client.h> 12#include <devif/queue_interface.h> 13#include <driverkit/iommu.h> 14 15#include <dev/e10k_q_dev.h> 16#include <dev/e10k_dev.h> 17#include <dev/e10k_vf_dev.h> 18#include <if/e10k_vf_defs.h> 19#include <if/e10k_vf_rpcclient_defs.h> 20 21#include "e10k_devif_vf.h" 22#include "e10k_queue.h" 23#include "debug.h" 24 25#define NUM_TX_DESC 2048 26#define NUM_RX_DESC 2048 27 28#define ETHHDR_LEN 14 29#define IPHDR_LEN 20 30#define UDPHDR_LEN 8 31 32 33// TODO only required for legacy interrupts 34struct e10k_queue* queues[128]; 35 36/* 37#define prnonz(x, d) \ 38 uint32_t x = e10k_vf_vf##x##_rd(d); \ 39 snprintf(str[cnt++], 32, #x "=%x \n", x); \ 40 41static void stats_dump(e10k_vf_t* d) 42{ 43 char str[256][32]; 44 int cnt = 0; 45 memset(str, 0, 256 * 32); 46 47 prnonz(ctrl, d); 48 prnonz(status, d); 49 prnonz(links, d); 50 prnonz(rxmemwrap, d); 51 prnonz(eicr, d); 52 prnonz(eics, d); 53 prnonz(eims, d); 54 prnonz(gprc, d); 55 prnonz(gptc, d); 56 57 if(cnt > 0) { 58 for(int i = 0; i < cnt; i++) { 59 printf("PF: %s ", str[i]); 60 } 61 printf("\n"); 62 } 63} 64 65static void print_packet(void* buf, size_t len) 66{ 67 printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ \n"); 68 uint8_t* b = (uint8_t*) buf; 69 for (int i = 0; i < len; i++) { 70 printf("%2X ", b[i]); 71 if ((i % 16) == 0) { 72 printf("\n 0x"); 73 } 74 } 75 76 printf(" \n @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ \n"); 77} 78*/ 79 80/******************************************************************************/ 81/* Misc functions */ 82 83static inline bool buf_use_tcpxsm(uint64_t flags) 84{ 85 return (flags & NETIF_TXFLAG_TCPCHECKSUM); 86} 87 88static inline bool buf_use_udpxsm(uint64_t flags) 89{ 90 return (flags & NETIF_TXFLAG_UDPCHECKSUM); 91} 92 93static inline bool buf_use_ipxsm(uint64_t flags) 94{ 95 return (flags & NETIF_TXFLAG_IPCHECKSUM) || 96 buf_use_tcpxsm(flags) || buf_use_udpxsm(flags); 97} 98 99static inline uint8_t buf_tcphdrlen(uint64_t flags) 100{ 101 return ((flags & NETIF_TXFLAG_TCPHDRLEN_MASK) >> 102 NETIF_TXFLAG_TCPHDRLEN_SHIFT) * 4; 103} 104 105static errval_t update_txtail(struct e10k_queue* q, size_t tail) 106{ 107 assert(q->d != NULL); 108 109 if (q->use_vf) { 110 e10k_vf_vftdt_wr(q->d, q->id, tail); 111 } else { 112 e10k_tdt_wr(q->d, q->id, tail); 113 } 114 return SYS_ERR_OK; 115} 116 117static errval_t update_rxtail(struct e10k_queue* q, size_t tail) 118{ 119 assert(q->d != NULL); 120 121 if (q->use_vf) { 122 e10k_vf_vfrdt_wr(q->d, q->id, tail); 123 } else { 124 e10k_rdt_1_wr(q->d, q->id, tail); 125 } 126 return SYS_ERR_OK; 127} 128 129 130static struct region_entry* get_region(struct e10k_queue* q, regionid_t rid) 131{ 132 struct region_entry* entry = q->regions; 133 while (entry != NULL) { 134 if (entry->rid == rid) { 135 return entry; 136 } 137 entry = entry->next; 138 } 139 return NULL; 140} 141 142static errval_t enqueue_tx_buf(struct e10k_queue* q, regionid_t rid, 143 genoffset_t offset, 144 genoffset_t length, 145 genoffset_t valid_data, 146 genoffset_t valid_length, 147 uint64_t flags) 148{ 149 if (e10k_queue_free_txslots(q) == 0) { 150 DEBUG_QUEUE("e10k_%d: Not enough space in TX ring, not adding buffer\n", 151 q->id); 152 // TODO better error 153 return NIC_ERR_ENQUEUE; 154 } 155 156 // Prepare checksum offload 157 // 158 struct region_entry* entry = get_region(q, rid); 159 assert(entry != NULL); 160 161 e10k_q_l4_type_t l4t = 0; 162 uint8_t l4len = 0; 163 164 165 DEBUG_QUEUE("TX Enqueuing offset=%lu valid_data=%lu phys=%lx txhwb=%d tx_tail=%zu tx_head=%zu" 166 " flags =%lx valid_length=%lu length=%lu\n", 167 offset, valid_data, entry->mem.devaddr + offset + valid_data, 168 (q->tx_hwb == NULL) ? 0 : *((uint32_t*)q->tx_hwb), q->tx_tail, 169 q->tx_head, flags, valid_length, length); 170 171 if (buf_use_ipxsm(flags)) { 172 173 if (buf_use_tcpxsm(flags)) { 174 l4t = e10k_q_tcp; 175 l4len = buf_tcphdrlen(flags); 176 } else if (buf_use_udpxsm(flags)) { 177 l4t = e10k_q_udp; 178 l4len = UDPHDR_LEN; 179 } 180 181 e10k_queue_add_txcontext(q, 0, ETHHDR_LEN, IPHDR_LEN, l4len, l4t); 182 183 lpaddr_t addr = 0; 184 addr = (lpaddr_t) entry->mem.devaddr + offset + valid_data; 185 e10k_queue_add_txbuf_ctx(q, addr, rid, offset, length, 186 valid_data, valid_length, flags, 187 valid_length, 0, true, l4len != 0); 188 } else { 189 lpaddr_t addr; 190 addr = (lpaddr_t) entry->mem.devaddr + offset + valid_data; 191 192 e10k_queue_add_txbuf(q, addr, rid, offset, length, valid_data, 193 valid_length, flags, 194 valid_length); 195 } 196 e10k_queue_bump_txtail(q); 197 return SYS_ERR_OK; 198} 199 200 201static errval_t enqueue_rx_buf(struct e10k_queue* q, regionid_t rid, 202 genoffset_t offset, 203 genoffset_t length, 204 genoffset_t valid_data, 205 genoffset_t valid_length, 206 uint64_t flags) 207{ 208 //DEBUG_QUEUE("Enqueueing RX buf \n"); 209 // check if there is space 210 if (e10k_queue_free_rxslots(q) == 0) { 211 DEBUG_QUEUE("e10k_%d: Not enough space in RX ring, not adding buffer\n", 212 q->id); 213 // TODO better error 214 return NIC_ERR_ENQUEUE; 215 } 216 217 // get virtual address of buffer 218 struct region_entry* entry = get_region(q, rid); 219 assert(entry != NULL); 220 221 lpaddr_t addr = 0; 222 addr = (lpaddr_t) entry->mem.devaddr + offset; 223 e10k_queue_add_rxbuf(q, addr, rid, offset, length, valid_data, 224 valid_length, flags); 225 226 e10k_queue_bump_rxtail(q); 227 return SYS_ERR_OK; 228} 229 230/******************************************************************************/ 231/* Queue functions */ 232 233static errval_t e10k_enqueue(struct devq* q, regionid_t rid, genoffset_t offset, 234 genoffset_t length, genoffset_t valid_data, 235 genoffset_t valid_length, uint64_t flags) 236{ 237 errval_t err; 238 239 240 struct e10k_queue* queue = (struct e10k_queue*) q; 241 if (flags & NETIF_RXFLAG) { 242 /* can not enqueue receive buffer larger than 2048 bytes */ 243 assert(length <= 2048); 244 245 err = enqueue_rx_buf(queue, rid, offset, length, valid_data, 246 valid_length, flags); 247 if (err_is_fail(err)) { 248 return err; 249 } 250 } else if (flags & NETIF_TXFLAG) { 251 252 assert(length <= 2048); 253 254 err = enqueue_tx_buf(queue, rid, offset, length, valid_data, 255 valid_length, flags); 256 if (err_is_fail(err)) { 257 return err; 258 } 259 } 260 261 return SYS_ERR_OK; 262} 263 264 265static errval_t e10k_dequeue(struct devq* q, regionid_t* rid, 266 genoffset_t* offset, genoffset_t* length, 267 genoffset_t* valid_data, 268 genoffset_t* valid_length, uint64_t* flags) 269{ 270 struct e10k_queue* que = (struct e10k_queue*) q; 271 int last; 272 errval_t err = SYS_ERR_OK; 273 274 if (!e10k_queue_get_txbuf(que, rid, offset, length, valid_data, 275 valid_length, flags)) { 276 err = DEVQ_ERR_QUEUE_EMPTY; 277 } else { 278 DEBUG_QUEUE("Queue %d sent offset=%lu valid_length=%lu \n", 279 que->id, *offset, *valid_length); 280 return SYS_ERR_OK; 281 } 282 283 if (!e10k_queue_get_rxbuf(que, rid, offset, length, valid_data, 284 valid_length, flags, &last)) { 285 err = DEVQ_ERR_QUEUE_EMPTY; 286 } else { 287 DEBUG_QUEUE("Queue %d received offset=%lu valid_length=%lu \n", 288 que->id, *offset, *valid_length); 289 return SYS_ERR_OK; 290 } 291 292 293 return err; 294} 295 296static errval_t e10k_register(struct devq* q, struct capref cap, regionid_t rid) 297{ 298 errval_t err; 299 struct e10k_queue* queue = (struct e10k_queue*) q; 300 301 struct frame_identity id; 302 err = frame_identify(cap, &id); 303 if (err_is_fail(err)) { 304 return err; 305 } 306 307 /* 308 void* va; 309 err = vspace_map_one_frame_attr(&va, id.bytes, cap, 310 VREGION_FLAGS_READ_WRITE, 311 NULL, NULL); 312 if (err_is_fail(err)) { 313 return err; 314 } 315 316 entry->mem.devaddr = id.bytes; 317 entry->mem.vbase = (lvaddr_t) va; 318 entry->mem.mem = cap; 319 */ 320 // keep track of regions since we need the virtual address ... 321 struct region_entry* entry = malloc(sizeof(struct region_entry)); 322 entry->rid = rid; 323 entry->next = NULL; 324 325 struct iommu_client* cl = e10k_vf_get_iommu_client(); 326 327 err = driverkit_iommu_vspace_map_cl(cl, cap, 328 VREGION_FLAGS_READ_WRITE, 329 &entry->mem); 330 if (err_is_fail(err)) { 331 free(entry); 332 return err; 333 } 334 335 DEBUG_QUEUE("register region id %d base=%lx \n", rid, entry->mem.devaddr); 336 // linked list of regions 337 struct region_entry* cur = queue->regions; 338 if (cur == NULL) { 339 queue->regions = entry; 340 return SYS_ERR_OK; 341 } 342 343 while (cur->next != NULL) { 344 cur = cur->next; 345 } 346 347 cur->next = entry; 348 349 DEBUG_QUEUE("registerd region id %d base=%p len=%ld \n", rid, 350 (void*) entry->mem.vbase, entry->mem.size); 351 352 return SYS_ERR_OK; 353} 354 355static errval_t e10k_deregister(struct devq* q, regionid_t rid) 356{ 357 return SYS_ERR_OK; 358} 359 360static errval_t e10k_control(struct devq* q, uint64_t cmd, uint64_t value, uint64_t *result) 361{ 362 struct e10k_queue* queue = (struct e10k_queue*) q; 363 *result = queue->mac; 364 return SYS_ERR_OK; 365} 366 367 368static errval_t e10k_notify(struct devq* q) 369{ 370 return SYS_ERR_OK; 371} 372 373static errval_t e10k_destroy(struct devq* queue) 374{ 375 struct e10k_queue* q = (struct e10k_queue*) queue; 376 free(q); 377 //TODO: rest of the cleanup 378 return SYS_ERR_OK; 379} 380 381/****************************************************************** 382 * Management functions 383 * 384 */ 385 386static void interrupt_cb(struct e10k_vf_binding *b, uint16_t qid) 387{ 388 struct e10k_queue* q = queues[qid]; 389 390 DEBUG_QUEUE("Interrupt on queue %d \n", qid); 391 if (q != b->st) { 392 debug_printf("STATE MISMATCH!\n %p %p\n", q, b->st); 393 q = b->st; 394 } 395 396 q->cb(q); 397} 398 399static struct e10k_vf_rx_vtbl rx_vtbl = { 400 .interrupt = interrupt_cb, 401}; 402 403static void bind_cb(void *st, errval_t err, struct e10k_vf_binding *b) 404{ 405 struct e10k_queue* q = (struct e10k_queue*) st; 406 assert(err_is_ok(err)); 407 408 DEBUG_QUEUE("Sucessfully connected to management interface\n"); 409 410 b->st = q; 411 q->binding = b; 412 b->rx_vtbl = rx_vtbl; 413 e10k_vf_rpc_client_init(q->binding); 414 q->bound = true; 415} 416 417/** Connect to the management interface */ 418static void connect_to_mngif(struct e10k_queue* q) 419{ 420 errval_t r; 421 iref_t iref; 422 423 q->bound = false; 424 char name[strlen("e10k_vf") + 2]; 425 426 // Build label for interal management service 427 sprintf(name, "%s%u", "e10k_vf", q->pci_function); 428 429 // Connect to service 430 r = nameservice_blocking_lookup(name, &iref); 431 assert(err_is_ok(r)); 432 433 r = e10k_vf_bind(iref, bind_cb, q, get_default_waitset(), 434 IDC_BIND_FLAGS_DEFAULT); 435 assert(err_is_ok(r)); 436 437 while (!q->bound) { 438 event_dispatch(get_default_waitset()); 439 } 440} 441 442/** Connect to the management interface */ 443static errval_t connect_to_mngif_with_ep(struct e10k_queue* q, struct capref ep) 444{ 445 errval_t err; 446 447 q->bound = false; 448 449 err = e10k_vf_bind_to_endpoint(ep, bind_cb, q, get_default_waitset(), 450 IDC_BIND_FLAGS_DEFAULT); 451 if (err_is_fail(err)) { 452 return err; 453 } 454 455 while (!q->bound) { 456 event_dispatch(get_default_waitset()); 457 } 458 return SYS_ERR_OK; 459} 460 461/********************************************************* 462 * Queue creation 463 */ 464 465static errval_t map_device_memory(struct e10k_queue* q, 466 struct capref regs) 467{ 468 469 struct frame_identity id = {.base = 0, .bytes = 0}; 470 errval_t err; 471 472 err = frame_identify(regs, &id); 473 if (err_is_fail(err)) { 474 return err; 475 } 476 477 void* va; 478 err = vspace_map_one_frame_attr(&va, id.bytes, regs, VREGION_FLAGS_READ_WRITE_NOCACHE, 479 NULL, NULL); 480 if (err_is_fail(err)) { 481 return err; 482 } 483 484 DEBUG_QUEUE("mapped %zu bytes at address %p\n", id.bytes, 485 va); 486 q->d = malloc(sizeof(e10k_t)); 487 assert(q->d != NULL); 488 e10k_initialize(q->d, (void*) va); 489 return SYS_ERR_OK; 490} 491 492// TODO mostly cleanup when fail 493errval_t e10k_queue_create(struct e10k_queue** queue, e10k_event_cb_t cb, struct capref* ep, 494 uint32_t bus, uint32_t function, uint32_t devid, uint32_t dev, 495 bool use_vf, bool interrupts, bool qzero) 496{ 497 498 errval_t err; 499 struct e10k_queue* q; 500 struct iommu_client* cl = NULL; 501 uint8_t vf_num; 502 // start VF driver 503 504 q = malloc(sizeof(struct e10k_queue)); 505 assert(q); 506 q->pci_function = 0; // TODO allow also function 1 507 q->use_msix = false; // TODO init MSI-X 508 509 // txhwb 510 if (use_vf) { 511 q->use_txhwb = false; 512 } else { 513 // TODO revert to true 514 q->use_txhwb = true; 515 } 516 q->cb = cb; 517 518 if (use_vf) { 519 //USER_PANIC("NOT YET WORKING \n"); 520 // Start VF 521 if (!e10k_vf_started()) { 522 err = e10k_init_vf_driver(ep, 0, 0, bus+1, dev+16, interrupts, &vf_num); 523 if (err_is_fail(err)) { 524 free(q); 525 *queue = NULL; 526 return err; 527 } 528 } 529 530 // If i can not create any more queues -> start new VF 531 if (!e10k_vf_can_create_queue()) { 532 err = e10k_init_vf_driver(ep, 0, 0, bus+1, dev+17, interrupts, &vf_num); 533 if (err_is_fail(err)) { 534 free(q); 535 return err; 536 } 537 } 538 539 q->use_vtd = true; 540 cl = e10k_vf_get_iommu_client(); 541 DEBUG_QUEUE("Assume enabled VT-d \n"); 542 } else { 543 q->use_vtd = false; 544 // need to set up communicaton to PF 545 if (ep == NULL || capref_is_null(*ep)) { 546 DEBUG_QUEUE("Connect to PF \n"); 547 connect_to_mngif(q); 548 } else { 549 DEBUG_QUEUE("Connect to PF using EP\n"); 550 err = connect_to_mngif_with_ep(q, *ep); 551 if (err_is_fail(err)) { 552 free(q); 553 return err; 554 } 555 } 556 } 557 558 559 // allocate memory for RX/TX rings 560 q->tx_ring_size = e10k_q_tdesc_adv_wb_size*NUM_TX_DESC; 561 562 DEBUG_QUEUE("Allocating TX queue memory\n"); 563 err = driverkit_iommu_mmap_cl(cl, q->tx_ring_size, VREGION_FLAGS_READ_WRITE, 564 &q->tx); 565 if (err_is_fail(err)) { 566 // TODO cleanup 567 return DEVQ_ERR_INIT_QUEUE; 568 } 569 570 571 DEBUG_QUEUE("Allocated TX queue memory is=%lu requested=%lu \n", q->tx.size, q->tx_ring_size); 572 DEBUG_QUEUE("Allocating RX queue memory\n"); 573 q->rx_ring_size = e10k_q_rdesc_adv_wb_size*NUM_RX_DESC; 574 err = driverkit_iommu_mmap_cl(cl, q->rx_ring_size, VREGION_FLAGS_READ_WRITE, 575 &q->rx); 576 if (err_is_fail(err)) { 577 // TODO cleanup 578 return DEVQ_ERR_INIT_QUEUE; 579 } 580 581 DEBUG_QUEUE("Allocated RX queue memory is=%lu requested=%lu \n", q->rx.size, q->rx_ring_size); 582 struct e10k_queue_ops ops = { 583 .update_txtail = update_txtail, 584 .update_rxtail = update_rxtail, 585 }; 586 587 if (q->use_txhwb) { 588 DEBUG_QUEUE("Allocating TX HWB queue memory\n"); 589 err = driverkit_iommu_mmap_cl(cl, BASE_PAGE_SIZE, VREGION_FLAGS_READ_WRITE, 590 &q->txhwb); 591 if (err_is_fail(err)) { 592 // TODO cleanup 593 return DEVQ_ERR_INIT_QUEUE; 594 } 595 memset((void*) q->txhwb.vbase, 0, sizeof(uint32_t)); 596 } 597 598 DEBUG_QUEUE("Init queue struct\n"); 599 e10k_queue_init(q, (void*) q->tx.vbase, NUM_TX_DESC, (void*) q->txhwb.vbase, 600 (void*) q->rx.vbase, NUM_RX_DESC, &ops); 601 602 DEBUG_QUEUE("Local queue init done\n"); 603 604 q->use_vf = use_vf; 605 q->use_irq = interrupts; 606 q->rx_frame = q->rx.mem; 607 q->tx_frame = q->tx.mem; 608 q->txhwb_frame = q->txhwb.mem; 609 610 // XXX:disable by default for now 611 q->use_rsc = false; 612 613 if (q->use_vf) { 614 err = e10k_vf_init_queue_hw(q, vf_num); 615 if (err_is_fail(err)) { 616 return err; 617 } 618 } else { 619 620 int qid; 621 errval_t err2; 622 struct capref regs; 623 624 if (q->use_irq) { 625 /* 626 err = pci_setup_inthandler(interrupt_handler, NULL, &vector); 627 assert(err_is_ok(err)); 628 core = disp_get_core_id(); 629 */ 630 // TODO setup MSI-X interrupts 631 } 632 633 // Inform card driver about new queue and get the registers/queue id 634 err = slot_alloc(®s); 635 if (err_is_fail(err)) { 636 return err; 637 } 638 639 err = slot_alloc(&(q->filter_ep)); 640 if (err_is_fail(err)) { 641 slot_free(regs); 642 return err; 643 } 644 645 err = q->binding->rpc_tx_vtbl.create_queue(q->binding, q->tx.mem, q->txhwb.mem, 646 q->rx.mem, 2048, disp_get_core_id(), 647 q->use_irq, q->use_msix, false, qzero, 648 &q->mac, &qid, 649 ®s, &q->filter_ep, &err2); 650 651 if (err_is_fail(err) || err_is_fail(err2)) { 652 DEBUG_QUEUE("e10k rpc error\n"); 653 return err_is_fail(err)? err: err2; 654 } 655 656 assert(qid >= 0); 657 q->id = (uint16_t)qid; 658 659 err = map_device_memory(q, regs); 660 if (err_is_fail(err)) { 661 DEBUG_QUEUE("e10k map device error\n"); 662 return err; 663 } 664 665 update_txtail(q, 0); 666 update_rxtail(q, 0); 667 668 } 669 670 err = devq_init(&q->q, false); 671 if (err_is_fail(err)) { 672 DEBUG_QUEUE("e10k devq_init error\n"); 673 return err; 674 } 675 676 q->q.f.enq = e10k_enqueue; 677 q->q.f.deq = e10k_dequeue; 678 q->q.f.reg = e10k_register; 679 q->q.f.dereg = e10k_deregister; 680 q->q.f.ctrl = e10k_control; 681 q->q.f.notify = e10k_notify; 682 q->q.f.destroy = e10k_destroy; 683 q->q.iommu = cl; 684 685 *queue = q; 686 queues[q->id] = q; 687 688 DEBUG_QUEUE("e10k queue init done\n"); 689 return SYS_ERR_OK; 690} 691 692 693void e10k_queue_get_netfilter_ep(struct e10k_queue* q, struct capref* ep) 694{ 695 *ep = q->filter_ep; 696} 697 698uint64_t e10k_queue_get_id(struct e10k_queue* q) 699{ 700 return q->id; 701} 702 703struct bench_ctl* e10k_get_benchmark_data(struct devq* q, uint8_t type) 704{ 705 return e10k_queue_get_benchmark_data((struct e10k_queue*) q, type); 706} 707 708