1/* 2 * Copyright (c) 2007-2011, ETH Zurich. 3 * All rights reserved. 4 * 5 * This file is distributed under the terms in the attached LICENSE file. 6 * If you do not find this file, copies can be found by writing to: 7 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 8 */ 9 10#include <barrelfish/barrelfish.h> 11#include <barrelfish/nameservice_client.h> 12#include <devif/queue_interface.h> 13#include <dev/e10k_q_dev.h> 14#include <dev/e10k_dev.h> 15#include <dev/e10k_vf_dev.h> 16#include <skb/skb.h> 17 18#include <if/e10k_vf_defs.h> 19#include <if/e10k_vf_rpcclient_defs.h> 20 21#include "e10k_devif_vf.h" 22#include "helper.h" 23#include "e10k_queue.h" 24#include "debug.h" 25 26#define NUM_TX_DESC 2048 27#define NUM_RX_DESC 2048 28 29#define ETHHDR_LEN 14 30#define IPHDR_LEN 20 31#define UDPHDR_LEN 8 32 33// for debugging 34static e10k_t* d; 35 36// TODO only required for legacy interrupts 37struct e10k_queue* queues[128]; 38 39/******************************************************************************/ 40/* Misc functions */ 41 42 43static inline bool buf_use_tcpxsm(uint64_t flags) 44{ 45 return (flags & NETIF_TXFLAG_TCPCHECKSUM); 46} 47 48static inline bool buf_use_udpxsm(uint64_t flags) 49{ 50 return (flags & NETIF_TXFLAG_UDPCHECKSUM); 51} 52 53static inline bool buf_use_ipxsm(uint64_t flags) 54{ 55 return (flags & NETIF_TXFLAG_IPCHECKSUM) || 56 buf_use_tcpxsm(flags) || buf_use_udpxsm(flags); 57} 58 59static inline uint8_t buf_tcphdrlen(uint64_t flags) 60{ 61 return ((flags & NETIF_TXFLAG_TCPHDRLEN_MASK) >> 62 NETIF_TXFLAG_TCPHDRLEN_SHIFT) * 4; 63} 64 65static errval_t update_txtail(struct e10k_queue* q, size_t tail) 66{ 67 assert(q->d != NULL); 68 69 if (q->use_vf) { 70 e10k_vf_vftdt_wr(q->d, q->id, tail); 71 } else { 72 e10k_tdt_wr(q->d, q->id, tail); 73 } 74 return SYS_ERR_OK; 75} 76 77static errval_t update_rxtail(struct e10k_queue* q, size_t tail) 78{ 79 assert(q->d != NULL); 80 81 if (q->use_vf) { 82 e10k_vf_vfrdt_wr(q->d, q->id, tail); 83 } else { 84 e10k_rdt_1_wr(q->d, q->id, tail); 85 } 86 return SYS_ERR_OK; 87} 88 89 90static struct region_entry* get_region(struct e10k_queue* q, regionid_t rid) 91{ 92 struct region_entry* entry = q->regions; 93 while (entry != NULL) { 94 if (entry->rid == rid) { 95 return entry; 96 } 97 entry = entry->next; 98 } 99 return NULL; 100} 101 102static errval_t enqueue_tx_buf(struct e10k_queue* q, regionid_t rid, 103 genoffset_t offset, 104 genoffset_t length, 105 genoffset_t valid_data, 106 genoffset_t valid_length, 107 uint64_t flags) 108{ 109 DEBUG_QUEUE("Enqueueing TX buf \n"); 110 111 if (e10k_queue_free_txslots(q) == 0) { 112 DEBUG_QUEUE("e10k_%d: Not enough space in TX ring, not adding buffer\n", 113 q->id); 114 // TODO better error 115 return NIC_ERR_ENQUEUE; 116 } 117 118 // Prepare checksum offload 119 // 120 struct region_entry* entry = get_region(q, rid); 121 assert(entry != NULL); 122 123 e10k_q_l4_type_t l4t = 0; 124 uint8_t l4len = 0; 125 126 if (buf_use_ipxsm(flags)) { 127 128 if (buf_use_tcpxsm(flags)) { 129 l4t = e10k_q_tcp; 130 l4len = buf_tcphdrlen(flags); 131 } else if (buf_use_udpxsm(flags)) { 132 l4t = e10k_q_udp; 133 l4len = UDPHDR_LEN; 134 } 135 136 e10k_queue_add_txcontext(q, 0, ETHHDR_LEN, IPHDR_LEN, l4len, l4t); 137 138 if (q->use_vtd) { 139 lpaddr_t addr = 0; 140 addr = (lpaddr_t) entry->virt + offset + valid_data; 141 e10k_queue_add_txbuf_ctx(q, addr, rid, offset, length, 142 valid_data, valid_length, flags, 143 valid_length, 0, true, l4len !=0); 144 } else { 145 lpaddr_t addr = 0; 146 addr = (lpaddr_t) entry->phys + offset + valid_data; 147 e10k_queue_add_txbuf_ctx(q, addr, rid, offset, length, 148 valid_data, valid_length, flags, 149 valid_length, 0, true, l4len != 0); 150 } 151 } else { 152 if (q->use_vtd) { 153 lvaddr_t addr = 0; 154 addr = (lvaddr_t) entry->virt + offset + valid_data; 155 e10k_queue_add_txbuf(q, addr, rid, offset, length, valid_data, 156 valid_length, flags, 157 valid_length); 158 } else { 159 lpaddr_t addr; 160 addr = (lpaddr_t) entry->phys + offset + valid_data; 161 162 e10k_queue_add_txbuf(q, addr, rid, offset, length, valid_data, 163 valid_length, flags, 164 valid_length); 165 } 166 } 167 e10k_queue_bump_txtail(q); 168 return SYS_ERR_OK; 169} 170 171 172static errval_t enqueue_rx_buf(struct e10k_queue* q, regionid_t rid, 173 genoffset_t offset, 174 genoffset_t length, 175 genoffset_t valid_data, 176 genoffset_t valid_length, 177 uint64_t flags) 178{ 179 //DEBUG_QUEUE("Enqueueing RX buf \n"); 180 // check if there is space 181 if (e10k_queue_free_rxslots(q) == 0) { 182 DEBUG_QUEUE("e10k_%d: Not enough space in RX ring, not adding buffer\n", 183 q->id); 184 // TODO better error 185 return NIC_ERR_ENQUEUE; 186 } 187 188 if (q->use_vtd) { 189 // get virtual address of buffer 190 struct region_entry* entry = get_region(q, rid); 191 assert(entry != NULL); 192 193 lpaddr_t addr = 0; 194 addr = (lpaddr_t) entry->virt + offset; 195 e10k_queue_add_rxbuf(q, addr, rid, offset, length, valid_data, 196 valid_length, flags); 197 } else { 198 // get virtual address of buffer 199 struct region_entry* entry = get_region(q, rid); 200 assert(entry != NULL); 201 202 lpaddr_t addr = 0; 203 addr = (lpaddr_t) entry->phys + offset; 204 e10k_queue_add_rxbuf(q, addr, rid, offset, length, valid_data, 205 valid_length, flags); 206 } 207 208 e10k_queue_bump_rxtail(q); 209 return SYS_ERR_OK; 210} 211 212/******************************************************************************/ 213/* Queue functions */ 214 215static errval_t e10k_enqueue(struct devq* q, regionid_t rid, genoffset_t offset, 216 genoffset_t length, genoffset_t valid_data, 217 genoffset_t valid_length, uint64_t flags) 218{ 219 errval_t err; 220 221 222 struct e10k_queue* queue = (struct e10k_queue*) q; 223 if (flags & NETIF_RXFLAG) { 224 /* can not enqueue receive buffer larger than 2048 bytes */ 225 assert(length <= 2048); 226 227 err = enqueue_rx_buf(queue, rid, offset, length, valid_data, 228 valid_length, flags); 229 if (err_is_fail(err)) { 230 return err; 231 } 232 } else if (flags & NETIF_TXFLAG) { 233 234 assert(length <= 2048); 235 236 DEBUG_QUEUE("Enqueuing offset=%lu valid_data=%lu txhwb=%d tx_tail=%zu tx_head=%zu \n", 237 offset, valid_data, (queue->tx_hwb == NULL) ? 0 : *((uint32_t*)queue->tx_hwb), queue->tx_tail, 238 queue->tx_head); 239 240 err = enqueue_tx_buf(queue, rid, offset, length, valid_data, 241 valid_length, flags); 242 if (err_is_fail(err)) { 243 return err; 244 } 245 } 246 247 return SYS_ERR_OK; 248} 249 250 251static errval_t e10k_dequeue(struct devq* q, regionid_t* rid, 252 genoffset_t* offset, genoffset_t* length, 253 genoffset_t* valid_data, 254 genoffset_t* valid_length, uint64_t* flags) 255{ 256 struct e10k_queue* que = (struct e10k_queue*) q; 257 int last; 258 errval_t err = SYS_ERR_OK; 259 260 if (!e10k_queue_get_txbuf(que, rid, offset, length, valid_data, 261 valid_length, flags)) { 262 err = DEVQ_ERR_QUEUE_EMPTY; 263 } else { 264 DEBUG_QUEUE("Queue %d sent offset=%lu valid_length=%lu transmit count %d\n", 265 que->id, *offset, *valid_length, e10k_vf_vfgptc_rd(que->d)); 266 return SYS_ERR_OK; 267 } 268 269 if (!e10k_queue_get_rxbuf(que, rid, offset, length, valid_data, 270 valid_length, flags, &last)) { 271 err = DEVQ_ERR_QUEUE_EMPTY; 272 } else { 273 DEBUG_QUEUE("Queue %d received offset=%lu valid_length=%lu \n", 274 que->id, *offset, *valid_length); 275 return SYS_ERR_OK; 276 } 277 278 279 return err; 280} 281 282static errval_t e10k_register(struct devq* q, struct capref cap, regionid_t rid) 283{ 284 errval_t err; 285 struct e10k_queue* queue = (struct e10k_queue*) q; 286 287 struct frame_identity id; 288 err = invoke_frame_identify(cap, &id); 289 if (err_is_fail(err)) { 290 return err; 291 } 292 293 void* va; 294 err = vspace_map_one_frame_attr(&va, id.bytes, cap, 295 VREGION_FLAGS_READ_WRITE, 296 NULL, NULL); 297 if (err_is_fail(err)) { 298 return err; 299 } 300 301 // keep track of regions since we need the virtual address ... 302 struct region_entry* entry = malloc(sizeof(struct region_entry)); 303 entry->rid = rid; 304 entry->cap = cap; 305 entry->phys = id.base; 306 entry->virt = (lvaddr_t)va; 307 entry->size = id.bytes; 308 entry->next = NULL; 309 310 // linked list of regions 311 struct region_entry* cur = queue->regions; 312 if (cur == NULL) { 313 queue->regions = entry; 314 return SYS_ERR_OK; 315 } 316 317 while (cur->next != NULL) { 318 cur = cur->next; 319 } 320 321 cur->next = entry; 322 323 return SYS_ERR_OK; 324} 325 326static errval_t e10k_deregister(struct devq* q, regionid_t rid) 327{ 328 return SYS_ERR_OK; 329} 330 331static errval_t e10k_control(struct devq* q, uint64_t cmd, uint64_t value, uint64_t *result) 332{ 333 struct e10k_queue* queue = (struct e10k_queue*) q; 334 *result = queue->mac; 335 return SYS_ERR_OK; 336} 337 338 339static errval_t e10k_notify(struct devq* q) 340{ 341 return SYS_ERR_OK; 342} 343 344static errval_t e10k_destroy(struct devq* queue) 345{ 346 struct e10k_queue* q = (struct e10k_queue*) queue; 347 free(q); 348 //TODO: rest of the cleanup 349 return SYS_ERR_OK; 350} 351 352/****************************************************************** 353 * Management functions 354 * 355 */ 356 357static void interrupt_cb(struct e10k_vf_binding *b, uint16_t qid) 358{ 359 struct e10k_queue* q = queues[qid]; 360 361 if (q != b->st) { 362 debug_printf("STATE MISMATCH!\n %p %p\n", q, b->st); 363 q = b->st; 364 } 365 366 q->cb(q); 367} 368 369static struct e10k_vf_rx_vtbl rx_vtbl = { 370 .interrupt = interrupt_cb, 371}; 372 373static void bind_cb(void *st, errval_t err, struct e10k_vf_binding *b) 374{ 375 struct e10k_queue* q = (struct e10k_queue*) st; 376 assert(err_is_ok(err)); 377 378 DEBUG_QUEUE("Sucessfully connected to management interface\n"); 379 380 b->st = q; 381 q->binding = b; 382 b->rx_vtbl = rx_vtbl; 383 e10k_vf_rpc_client_init(q->binding); 384 q->bound = true; 385} 386 387/** Connect to the management interface */ 388static void connect_to_mngif(struct e10k_queue* q) 389{ 390 errval_t r; 391 iref_t iref; 392 393 q->bound = false; 394 char name[strlen("e10k_vf") + 2]; 395 396 // Build label for interal management service 397 sprintf(name, "%s%u", "e10k_vf", q->pci_function); 398 399 // Connect to service 400 DEBUG_QUEUE("Looking up management interface (%s)\n", name); 401 r = nameservice_blocking_lookup(name, &iref); 402 assert(err_is_ok(r)); 403 404 DEBUG_QUEUE("Binding to management interface\n"); 405 r = e10k_vf_bind(iref, bind_cb, q, get_default_waitset(), 406 IDC_BIND_FLAGS_DEFAULT); 407 assert(err_is_ok(r)); 408 409 while (!q->bound) { 410 event_dispatch(get_default_waitset()); 411 } 412} 413 414/********************************************************* 415 * Queue creation 416 */ 417 418static errval_t map_device_memory(struct e10k_queue* q, 419 struct capref regs) 420{ 421 422 struct frame_identity id = {.base = 0, .bytes = 0}; 423 errval_t err; 424 425 err = invoke_frame_identify(regs, &id); 426 if (err_is_fail(err)) { 427 return err; 428 } 429 430 void* va; 431 err = vspace_map_one_frame_attr(&va, id.bytes, regs, 432 VREGION_FLAGS_READ_WRITE_NOCACHE, 433 NULL, NULL); 434 if (err_is_fail(err)) { 435 return err; 436 } 437 438 439 DEBUG_QUEUE("mapped %zu bytes at address %p\n", id.bytes, va); 440 q->d = malloc(sizeof(e10k_t)); 441 assert(q->d != NULL); 442 e10k_initialize(q->d, va); 443 d = q->d; 444 return SYS_ERR_OK; 445} 446 447// TODO mostly cleanup when fail 448errval_t e10k_queue_create(struct e10k_queue** queue, e10k_event_cb_t cb, 449 uint32_t bus, uint32_t function, uint32_t devid, uint32_t dev, 450 bool use_vf, bool interrupts, bool qzero) 451{ 452 453 errval_t err; 454 struct e10k_queue* q; 455 // start VF driver 456 457 q = malloc(sizeof(struct e10k_queue)); 458 q->pci_function = 0; // TODO allow also function 1 459 460 // txhwb 461 if (use_vf) { 462 q->use_txhwb = false; 463 } else { 464 q->use_txhwb = true; 465 } 466 q->cb = cb; 467 468 if (use_vf) { 469 //USER_PANIC("NOT YET WORKING \n"); 470 // Start VF 471 if (!e10k_vf_started()) { 472 err = e10k_init_vf_driver(0, 0, bus+1, dev+16, devid, interrupts); 473 if (err_is_fail(err)) { 474 return err; 475 } 476 } 477 478 // If i can not create any more queues -> start new VF 479 if (!e10k_vf_can_create_queue()) { 480 err = e10k_init_vf_driver(0, 0, bus+1, dev+16, devid, interrupts); 481 if (err_is_fail(err)) { 482 return err; 483 } 484 } 485 486 err = skb_client_connect(); 487 assert(err_is_ok(err)); 488 err = skb_execute_query("vtd_enabled(0,_)."); 489 if (err_is_fail(err)) { 490 DEBUG_QUEUE("Assume disabled VT-d \n"); 491 q->use_vtd = false; 492 } else { 493 DEBUG_QUEUE("Assume enabled VT-d \n"); 494 q->use_vtd = true; 495 } 496 497 } else { 498 q->use_vtd = false; 499 // need to set up communicaton to PF 500 connect_to_mngif(q); 501 } 502 503 // allocate memory for RX/TX rings 504 struct capref tx_frame; 505 size_t tx_size = e10k_q_tdesc_adv_wb_size*NUM_TX_DESC; 506 void* tx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, tx_size, &tx_frame); 507 if (tx_virt == NULL) { 508 return DEVQ_ERR_INIT_QUEUE; 509 } 510 511 struct capref rx_frame; 512 size_t rx_size = e10k_q_rdesc_adv_wb_size*NUM_RX_DESC; 513 void* rx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, rx_size, &rx_frame); 514 if (rx_virt == NULL) { 515 return DEVQ_ERR_INIT_QUEUE; 516 } 517 518 struct e10k_queue_ops ops = { 519 .update_txtail = update_txtail, 520 .update_rxtail = update_rxtail, 521 }; 522 523 struct capref txhwb_frame; 524 void* txhwb_virt = NULL; 525 526 if (q->use_txhwb) { 527 txhwb_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, BASE_PAGE_SIZE, 528 &txhwb_frame); 529 if (txhwb_virt == NULL) { 530 return DEVQ_ERR_INIT_QUEUE; 531 } 532 memset(txhwb_virt, 0, sizeof(uint32_t)); 533 } 534 535 e10k_queue_init(q, tx_virt, NUM_TX_DESC, txhwb_virt, 536 rx_virt, NUM_RX_DESC, &ops); 537 538 DEBUG_QUEUE("Local queue init done\n"); 539 540 q->use_vf = use_vf; 541 q->rx_frame = rx_frame; 542 q->tx_frame = tx_frame; 543 q->txhwb_frame = txhwb_frame; 544 q->use_irq = interrupts; 545 546 // XXX:disable by default for now 547 q->use_rsc = false; 548 549 if (q->use_vf) { 550 err = e10k_vf_init_queue_hw(q); 551 if (err_is_fail(err)) { 552 return err; 553 } 554 } else { 555 556 int qid; 557 errval_t err2; 558 struct capref regs; 559 560 if (q->use_irq) { 561 /* 562 err = pci_setup_inthandler(interrupt_handler, NULL, &vector); 563 assert(err_is_ok(err)); 564 core = disp_get_core_id(); 565 */ 566 // TODO setup MSI-X interrupts 567 } 568 569 // Inform card driver about new queue and get the registers/queue id 570 err = slot_alloc(®s); 571 if (err_is_fail(err)) { 572 return err; 573 } 574 575 err = q->binding->rpc_tx_vtbl.create_queue(q->binding, tx_frame, txhwb_frame, 576 rx_frame, 2048, q->msix_intvec, 577 q->msix_intdest, q->use_irq, false, qzero, 578 &q->mac, &qid, 579 ®s, &err2); 580 if (err_is_fail(err) || err_is_fail(err2)) { 581 DEBUG_QUEUE("e10k rpc error\n"); 582 return err_is_fail(err)? err: err2; 583 } 584 585 assert(qid >= 0); 586 q->id = (uint16_t)qid; 587 588 err = map_device_memory(q, regs); 589 if (err_is_fail(err)) { 590 DEBUG_QUEUE("e10k map device error\n"); 591 return err; 592 } 593 594 update_txtail(q, 0); 595 update_rxtail(q, 0); 596 597 } 598 599 err = devq_init(&q->q, false); 600 if (err_is_fail(err)) { 601 DEBUG_QUEUE("e10k devq_init error\n"); 602 return err; 603 } 604 605 q->q.f.enq = e10k_enqueue; 606 q->q.f.deq = e10k_dequeue; 607 q->q.f.reg = e10k_register; 608 q->q.f.dereg = e10k_deregister; 609 q->q.f.ctrl = e10k_control; 610 q->q.f.notify = e10k_notify; 611 q->q.f.destroy = e10k_destroy; 612 613 614 *queue = q; 615 queues[q->id] = q; 616 617 DEBUG_QUEUE("e10k queue init done\n"); 618 return SYS_ERR_OK; 619} 620 621uint64_t e10k_queue_get_id(struct e10k_queue* q) 622{ 623 return q->id; 624} 625