1/* 2 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich. 3 * All rights reserved. 4 * 5 * This file is distributed under the terms in the attached LICENSE file. 6 * If you do not find this file, copies can be found by writing to: 7 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group. 8 */ 9 10#include <stdlib.h> 11#include <stdio.h> 12#include <barrelfish/barrelfish.h> 13#include <barrelfish/waitset.h> 14#include <barrelfish/deferred.h> 15#include <barrelfish/nameservice_client.h> 16#include <devif/queue_interface.h> 17#include <if/sfn5122f_devif_defs.h> 18#include <if/sfn5122f_devif_defs.h> 19#include <devif/backends/net/sfn5122f_devif.h> 20#include <devif/queue_interface_backend.h> 21#include "hw_queue.h" 22#include "helper.h" 23 24//#define DEBUG_SFN 25#ifdef DEBUG_SFN 26 #define DEBUG_QUEUE(x...) debug_printf("sfn5122f_q : " x) 27#else 28 #define DEBUG_QUEUE(x...) do {} while (0) 29#endif 30 31//#define DELAY 1 32 33// TX Queue 34#define TX_ENTRIES 4096 35#define RX_ENTRIES 4096 36#define EV_ENTRIES 32768 37 38STATIC_ASSERT((TX_ENTRIES & (TX_ENTRIES - 1)) == 0, "must be a power of two"); 39STATIC_ASSERT((RX_ENTRIES & (RX_ENTRIES - 1)) == 0, "must be a power of two"); 40STATIC_ASSERT((EV_ENTRIES & (EV_ENTRIES - 1)) == 0, "must be a power of two"); 41 42 43// Event Queue 44#define EV_CODE_RX 0 45#define EV_CODE_TX 2 46#define EV_CODE_DRV 5 47#define EV_CODE_DRV_GEN 7 48#define EV_CODE_USER 8 49#define EV_CODE_MCDI 12 50#define EV_CODE_GLOBAL 6 51#define EV_CODE_NONE 15 52 53#define BUF_SIZE 4096 54 55/* for each TX/RX entry one entry plus an additonal 2 for mcdi completion 56and link state events */ 57 58struct sfn5122f_queue* queues[1024]; 59 60 61/** Misc */ 62static errval_t update_rxtail(struct sfn5122f_queue* q, size_t tail) 63{ 64 assert(q->device != NULL); 65 uint64_t reg = 0; 66 67 q->rx_batch_size++; 68 69 if (q->rx_batch_size > 31) { 70 /* Write to this register is very very expensive (2500 cycles +) 71 So we batch the updates together*/ 72 reg = sfn5122f_rx_desc_upd_reg_hi_rx_desc_wptr_insert(reg, tail); 73 /* don't want to push an additional rx descriptor with the write pointer */ 74 reg = sfn5122f_rx_desc_upd_reg_hi_rx_desc_push_cmd_insert(reg, 0); 75 /* the lower register will be ignored */ 76 sfn5122f_rx_desc_upd_reg_lo_wr(q->device, q->id, 0); 77 sfn5122f_rx_desc_upd_reg_hi_wr(q->device, q->id, reg); 78 q->rx_batch_size = 0; 79 } 80 81 return SYS_ERR_OK; 82} 83 84static errval_t update_txtail(struct sfn5122f_queue* q, size_t tail) 85{ 86 assert(q->device != NULL); 87 uint64_t reg = 0; 88 reg = sfn5122f_tx_desc_upd_reg_hi_tx_desc_wptr_insert(reg, tail); 89 /* don't want to push an additional tx descriptor with the write pointer */ 90 reg = sfn5122f_tx_desc_upd_reg_hi_tx_desc_push_cmd_insert(reg, 0); 91 reg = sfn5122f_tx_desc_upd_reg_hi_tx_desc_insert(reg, 0); 92 93 /* the lower register will be ignored */ 94 sfn5122f_tx_desc_upd_reg_lo_wr(q->device, q->id, 0); 95 sfn5122f_tx_desc_upd_reg_hi_wr(q->device, q->id, reg); 96 return SYS_ERR_OK; 97} 98 99static void interrupt_cb(struct sfn5122f_devif_binding *b, uint16_t qid) 100{ 101 struct sfn5122f_queue* q = queues[qid]; 102 103 if (q != b->st) { 104 debug_printf("STATE MISMATCH!\n %p %p\n", q, b->st); 105 q = b->st; 106 } 107 108 q->cb(q); 109} 110 111static struct sfn5122f_devif_rx_vtbl rx_vtbl = { 112 .interrupt = interrupt_cb, 113}; 114 115static void bind_cb(void *st, errval_t err, struct sfn5122f_devif_binding *b) 116{ 117 118 DEBUG_QUEUE("binding CB \n"); 119 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) st; 120 b->st = queue; 121 b->rx_vtbl = rx_vtbl; 122 // Initi RPC client 123 124 queue->b = b; 125 sfn5122f_devif_rpc_client_init(queue->b); 126 queue->bound = true; 127} 128 129 130static errval_t sfn5122f_register(struct devq* q, struct capref cap, 131 regionid_t rid) 132{ 133 uint64_t buftbl_idx = 0; 134 errval_t err, err2; 135 struct frame_identity id; 136 137 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) q; 138 139 if (queue->userspace) { 140 err = queue->b->rpc_tx_vtbl.register_region(queue->b, queue->id, cap, 141 &buftbl_idx, &err2); 142 if (err_is_fail(err) || err_is_fail(err2)) { 143 err = err_is_fail(err) ? err: err2; 144 return err; 145 } 146 } 147 148 err = frame_identify(cap, &id); 149 if (err_is_fail(err)) { 150 return err; 151 } 152 153 // Setup datastructure for translating region ID to buffer table entry id 154 // Currently only a linked list 155 struct region_entry* entry = malloc(sizeof(struct region_entry)); 156 entry->rid = rid; 157 entry->phys = id.base; 158 entry->size = id.bytes; 159 entry->cap = cap; 160 entry->buftbl_idx = buftbl_idx; 161 entry->next = NULL; 162 163 struct region_entry* cur = queue->regions; 164 165 if (cur == NULL) { 166 queue->regions = entry; 167 return SYS_ERR_OK; 168 } 169 170 while (cur->next != NULL) { 171 cur = cur->next; 172 } 173 174 cur->next = entry; 175 176 DEBUG_QUEUE("Region %d registered \n", rid); 177 return SYS_ERR_OK; 178} 179 180static errval_t sfn5122f_deregister(struct devq* q, regionid_t rid) 181{ 182 errval_t err, err2; 183 184 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) q; 185 186 // find region and translate to buftlb entry 187 struct region_entry* cur = queue->regions; 188 189 if (cur == NULL) { 190 return DEVQ_ERR_INVALID_REGION_ARGS; 191 } 192 193 while (cur->next != NULL && cur->rid != rid) { 194 cur = cur->next; 195 } 196 197 // do rpc do inform carddriver to remove buftbl entries 198 if (queue->userspace) { 199 err = queue->b->rpc_tx_vtbl.deregister_region(queue->b, cur->buftbl_idx, 200 cur->size, &err2); 201 if (err_is_fail(err) || err_is_fail(err2)) { 202 err = err_is_fail(err) ? err: err2; 203 return err; 204 } 205 } 206 207 return SYS_ERR_OK; 208} 209 210 211static errval_t sfn5122f_control(struct devq* q, uint64_t cmd, uint64_t value, 212 uint64_t *result) 213{ 214 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) q; 215 *result = queue->mac; 216 DEBUG_QUEUE("Control cmd=%lu value=%lu \n", cmd, value); 217 return SYS_ERR_OK; 218} 219 220 221static errval_t sfn5122f_notify(struct devq* q) 222{ 223 DEBUG_QUEUE("Notify \n"); 224 return SYS_ERR_OK; 225} 226 227static errval_t enqueue_rx_buf(struct sfn5122f_queue* q, regionid_t rid, 228 genoffset_t offset, genoffset_t length, 229 genoffset_t valid_data, genoffset_t valid_length, 230 uint64_t flags) 231{ 232 // check if there is space 233 234 if (sfn5122f_queue_free_rxslots(q) == 0) { 235 DEBUG_QUEUE("SFN5122F_%d: Not enough space in RX ring, not adding buffer\n", 236 q->id); 237 return DEVQ_ERR_QUEUE_FULL; 238 } 239 240 // find region 241 242 struct region_entry* entry = q->regions; 243 244 // If regions already empty -> return error 245 if (entry == NULL) { 246 return DEVQ_ERR_INVALID_REGION_ARGS; 247 } 248 249 while((entry->next != NULL) && (entry->rid != rid)) { 250 entry = entry->next; 251 } 252 253 if (entry == NULL) { 254 return DEVQ_ERR_INVALID_REGION_ARGS; 255 } 256 257 if (q->userspace) { 258 // compute buffer table entry of the rx buffer and the within it offset 259 uint64_t buftbl_idx = entry->buftbl_idx + (offset/BUF_SIZE); 260 uint16_t b_off = offset & 0x00000FFF; 261 262 DEBUG_QUEUE("RX_BUF tbl_idx=%lu offset=%d flags=%lu \n", 263 buftbl_idx, b_off, flags); 264 // still in the same buffer table entry 265 assert(buftbl_idx == (entry->buftbl_idx + ((offset+length-1)/BUF_SIZE))); 266 sfn5122f_queue_add_user_rxbuf_devif(q, buftbl_idx, b_off, 267 rid, offset, length, valid_data, 268 valid_length, flags); 269 } else { 270 sfn5122f_queue_add_rxbuf_devif(q, entry->phys + offset, rid, offset, length, 271 valid_data, valid_length, flags); 272 273 } 274 275 update_rxtail(q, q->rx_tail); 276 return SYS_ERR_OK; 277} 278 279static errval_t enqueue_tx_buf(struct sfn5122f_queue* q, regionid_t rid, 280 genoffset_t offset, genoffset_t length, 281 genoffset_t valid_data, genoffset_t valid_length, 282 uint64_t flags) 283{ 284 DEBUG_QUEUE("Enqueueing TX buf\n"); 285 // check if there is space 286 if (sfn5122f_queue_free_txslots(q) == 0) { 287 printf("SFN5122F_%d: Not enough space in TX ring, not adding buffer\n", 288 q->id); 289 return DEVQ_ERR_QUEUE_FULL; 290 } 291 292 // find region 293 struct region_entry* entry = q->regions; 294 295 if (entry == NULL) { 296 return DEVQ_ERR_INVALID_REGION_ARGS; 297 } 298 299 while((entry->next != NULL) && (entry->rid != rid)) { 300 entry = entry->next; 301 } 302 303 if (entry == NULL) { 304 return DEVQ_ERR_INVALID_REGION_ARGS; 305 } 306 307 308 if (q->userspace) { 309 // compute buffer table entry of the rx buffer and the within it offset 310 uint64_t buftbl_idx = entry->buftbl_idx + (offset/BUF_SIZE); 311 uint16_t b_off = offset & 0x00000FFF; 312 313 314 DEBUG_QUEUE("TX_BUF tbl_idx=%lu offset=%d flags=%lx \n", buftbl_idx, b_off, 315 flags); 316 // still in the same buffer table entry 317 assert(buftbl_idx == (entry->buftbl_idx + ((offset+length-1)/BUF_SIZE))); 318 sfn5122f_queue_add_user_txbuf_devif(q, buftbl_idx, b_off, 319 rid, offset, length, valid_data, 320 valid_length, flags); 321 322 } else { 323 324 DEBUG_QUEUE("TX_BUF phys=%zu \n", entry->phys + offset); 325 sfn5122f_queue_add_txbuf_devif(q, entry->phys + offset, rid, offset, 326 length, valid_data, valid_length, 327 flags); 328 } 329 update_txtail(q, q->tx_tail); 330 return SYS_ERR_OK; 331} 332 333static errval_t sfn5122f_enqueue(struct devq* q, regionid_t rid, 334 genoffset_t offset, genoffset_t length, 335 genoffset_t valid_data, genoffset_t valid_length, 336 uint64_t flags) 337{ 338 errval_t err; 339 340 341 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) q; 342 if (flags & NETIF_RXFLAG) { 343 /* can not enqueue receive buffer larger than 2048 bytes */ 344 assert(length <= 2048); 345 346 err = enqueue_rx_buf(queue, rid, offset, length, valid_data, valid_length, 347 flags); 348 if (err_is_fail(err)) { 349 return err; 350 } 351 } else if (flags & NETIF_TXFLAG) { 352 assert(length <= BASE_PAGE_SIZE); 353 354 err = enqueue_tx_buf(queue, rid, offset, length, valid_data, valid_length, 355 flags); 356 if (err_is_fail(err)) { 357 return err; 358 } 359 } else { 360 printf("Unknown buffer flags \n"); 361 return NIC_ERR_ENQUEUE; 362 } 363 364 return SYS_ERR_OK; 365} 366 367static errval_t sfn5122f_dequeue(struct devq* q, regionid_t* rid, genoffset_t* offset, 368 genoffset_t* length, genoffset_t* valid_data, 369 genoffset_t* valid_length, uint64_t* flags) 370{ 371 uint8_t ev_code; 372 errval_t err = DEVQ_ERR_QUEUE_EMPTY; 373 374 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) q; 375 376 //sfn5122f_evq_rptr_reg_wr(queue->device, queue->id, queue->ev_head); 377 //__sync_synchronize(); 378 379 if (queue->num_left > 0) { 380 *rid = queue->bufs[queue->last_deq].rid; 381 *offset = queue->bufs[queue->last_deq].offset; 382 *flags = queue->bufs[queue->last_deq].flags; 383 *valid_length = queue->bufs[queue->last_deq].valid_length; 384 *valid_data = queue->bufs[queue->last_deq].valid_data; 385 *length = queue->bufs[queue->last_deq].length; 386 queue->num_left--; 387 queue->last_deq++; 388 return SYS_ERR_OK; 389 } 390 391 while(true) { 392 ev_code = sfn5122f_get_event_code(queue); 393 switch(ev_code){ 394 case EV_CODE_RX: 395 // TODO multiple packets 396 err = sfn5122f_queue_handle_rx_ev_devif(queue, rid, offset, length, 397 valid_data, valid_length, 398 flags); 399 400 DEBUG_QUEUE("RX_EV Q_ID: %d len %ld OK %s \n", queue->id, *valid_length, 401 err_getstring(err)); 402 403 sfn5122f_queue_bump_evhead(queue); 404 405 if (err_is_fail(err)) { 406 debug_printf("enqueue again: rid=%u, off=%lx\n", *rid, *offset); 407 err = enqueue_rx_buf(queue, *rid, *offset, *length, 408 *valid_data, *valid_length, 409 *flags); 410 if (err_is_fail(err)) { 411 printf("Error receiving packet, could not enqueue buffer\n"); 412 /* we need to return the buffer here, and let the networkstack 413 * deal with it */ 414 return SYS_ERR_OK; 415 } 416 417 /* the packet has been discarded and enqueued successfully, 418 * return emtpy queue */ 419 err = DEVQ_ERR_QUEUE_EMPTY; 420 } else { 421 assert(*valid_length > 0); 422 return SYS_ERR_OK; 423 } 424 break; 425 case EV_CODE_TX: 426 err = sfn5122f_queue_handle_tx_ev_devif(queue, rid, offset, length, 427 valid_data, valid_length, 428 flags); 429 if (*flags & NETIF_RXFLAG) { 430 printf("HUH: reiceived rx buffer in tx event???\n"); 431 } 432 433 if (err_is_ok(err)) { 434 DEBUG_QUEUE("TX EVENT OK %d \n", queue->id); 435 } else { 436 DEBUG_QUEUE("TX EVENT ERR %d \n", queue->id); 437 } 438 439 sfn5122f_queue_bump_evhead(queue); 440 return SYS_ERR_OK; 441 case EV_CODE_DRV: 442 DEBUG_QUEUE("DRIVER EVENT %d\n", queue->id); 443 sfn5122f_handle_drv_ev(queue, queue->id); 444 sfn5122f_queue_bump_evhead(queue); 445 break; 446 case EV_CODE_DRV_GEN: 447 DEBUG_QUEUE("DRIVER GENERATED EVENT \n"); 448 sfn5122f_queue_bump_evhead(queue); 449 break; 450 case EV_CODE_USER: 451 DEBUG_QUEUE("USER EVENT \n"); 452 sfn5122f_queue_bump_evhead(queue); 453 break; 454 case EV_CODE_MCDI: 455 DEBUG_QUEUE("MCDI EVENT \n"); 456 sfn5122f_queue_handle_mcdi_event(queue); 457 sfn5122f_queue_bump_evhead(queue); 458 break; 459 case EV_CODE_GLOBAL: 460 DEBUG_QUEUE("GLOBAL EVENT \n"); 461 sfn5122f_queue_bump_evhead(queue); 462 break; 463 case EV_CODE_NONE: 464 if(queue->use_interrupts || ((queue->ev_head & ((EV_ENTRIES / 8) - 1)) == 0)) { 465 sfn5122f_evq_rptr_reg_wr(queue->device, queue->id, queue->ev_head); 466 } 467 468 return err; 469 } 470 } 471 472 return err; 473} 474 475static errval_t sfn5122f_destroy(struct devq* queue) 476{ 477 errval_t err, err2; 478 struct sfn5122f_queue* q; 479 480 q = (struct sfn5122f_queue*) queue; 481 482 err = q->b->rpc_tx_vtbl.destroy_queue(q->b, q->id, &err2); 483 if (err_is_fail(err) || err_is_fail(err2)) { 484 err = err_is_fail(err) ? err: err2; 485 return err; 486 } 487 488 err = vspace_unmap(q->device_va); 489 if (err_is_fail(err)) { 490 return err; 491 } 492 493 free(q->device); 494 free(q->b); 495 496 err = sfn5122f_queue_free(q); 497 if (err_is_fail(err)) { 498 return err; 499 } 500 501 return SYS_ERR_OK; 502} 503 504 505 506 507/* 508static void interrupt_handler(void* arg) 509{ 510 struct sfn5122f_queue* queue = (struct sfn5122f_queue*) arg; 511 512 queue->cb(queue); 513} 514*/ 515 516/** 517 * Public functions 518 * 519 */ 520 521errval_t sfn5122f_queue_create(struct sfn5122f_queue** q, sfn5122f_event_cb_t cb, 522 struct capref* ep, bool userlevel, bool interrupts, 523 bool qzero) 524{ 525 DEBUG_QUEUE("create called \n"); 526 527 errval_t err; 528 //struct capref tx_frame, rx_frame, ev_frame; 529 struct capref frame; 530 //size_t tx_size, rx_size, ev_size; 531 size_t total_size; 532 void *tx_virt, *rx_virt, *ev_virt; 533 struct sfn5122f_queue* queue; 534 struct frame_identity id; 535 536 struct sfn5122f_queue_ops ops = { 537 .update_txtail = update_txtail, 538 .update_rxtail = update_rxtail 539 }; 540 541 /* Allocate memory for descriptor rings 542 No difference for userspace networking*/ 543 total_size = sizeof(uint64_t)*(TX_ENTRIES + RX_ENTRIES + EV_ENTRIES); 544 tx_virt = alloc_map_frame(VREGION_FLAGS_READ_WRITE, total_size, &frame); 545 if (tx_virt == NULL) { 546 return DEVQ_ERR_INIT_QUEUE; 547 } 548 549 rx_virt = tx_virt + (sizeof(uint64_t) *TX_ENTRIES); 550 ev_virt = rx_virt + (sizeof(uint64_t) *RX_ENTRIES); 551 552 DEBUG_QUEUE("queue init \n"); 553 // Init queue 554 queue = sfn5122f_queue_init(tx_virt, TX_ENTRIES, rx_virt, RX_ENTRIES, 555 ev_virt, EV_ENTRIES, &ops, userlevel); 556 557 queue->frame = frame; 558 queue->bound = false; 559 queue->cb = cb; 560 queue->use_interrupts = interrupts; 561 562 563 if (ep == NULL || capref_is_null(*ep)) { 564 iref_t iref; 565 const char *name = "sfn5122f_sfn5122fmng_devif"; 566 567 // Connect to solarflare card driver 568 err = nameservice_blocking_lookup(name, &iref); 569 if (err_is_fail(err)) { 570 return err; 571 } 572 573 DEBUG_QUEUE("binding \n"); 574 err = sfn5122f_devif_bind(iref, bind_cb, queue, get_default_waitset(), 575 1); 576 if (err_is_fail(err)) { 577 return err; 578 } 579 } else { 580 DEBUG_QUEUE("binding \n"); 581 err = sfn5122f_devif_bind_to_endpoint(*ep, bind_cb, queue, 582 get_default_waitset(), 1); 583 if (err_is_fail(err)) { 584 return err; 585 } 586 } 587 // wait until bound 588 while(!queue->bound) { 589 event_dispatch(get_default_waitset()); 590 } 591 592 DEBUG_QUEUE("bound \n"); 593 594 errval_t err2; 595 struct capref regs; 596 597 // Inform card driver about new queue and get the registers/queue id 598 err = slot_alloc(®s); 599 if (err_is_fail(err)) { 600 return err; 601 } 602 603 err = slot_alloc(&queue->filter_ep); 604 if (err_is_fail(err)) { 605 return err; 606 } 607 608 609 queue->core = disp_get_core_id(); 610 611 if (!interrupts) { 612 printf("Solarflare queue used in polling mode (default %d) \n", qzero); 613 err = queue->b->rpc_tx_vtbl.create_queue(queue->b, frame, userlevel, 614 interrupts, qzero, 615 queue->core, 0, &queue->mac ,&queue->id, 616 &queue->filter_ep, ®s, &err2); 617 if (err_is_fail(err) || err_is_fail(err2)) { 618 err = err_is_fail(err) ? err: err2; 619 printf("Failed to create queue in Driver: %s\n", err_getstring(err)); 620 return err; 621 } 622 } else { 623 printf("Solarflare queue used in interrupt mode mode \n"); 624 625 err = queue->b->rpc_tx_vtbl.create_queue(queue->b, frame, userlevel, 626 interrupts, qzero, queue->core, 627 queue->vector, &queue->mac, 628 &queue->id, &queue->filter_ep, 629 ®s, &err2); 630 if (err_is_fail(err) || err_is_fail(err2)) { 631 err = err_is_fail(err) ? err: err2; 632 printf("Registering interrupt failed, continueing in polling mode \n"); 633 } 634 } 635 636 DEBUG_QUEUE("rpc done \n"); 637 638 err = frame_identify(regs, &id); 639 if (err_is_fail(err)) { 640 return err; 641 } 642 643 err = vspace_map_one_frame_attr(&queue->device_va, id.bytes, regs, 644 VREGION_FLAGS_READ_WRITE, NULL, NULL); 645 if (err_is_fail(err)) { 646 return err; 647 } 648 649 650 DEBUG_QUEUE("mapped \n"); 651 queue->device = malloc(sizeof(sfn5122f_t)); 652 sfn5122f_initialize(queue->device, queue->device_va); 653 654 err = devq_init(&queue->q, false); 655 if (err_is_fail(err)) { 656 return err; 657 } 658 659 queue->q.f.enq = sfn5122f_enqueue; 660 queue->q.f.deq = sfn5122f_dequeue; 661 queue->q.f.reg = sfn5122f_register; 662 queue->q.f.dereg = sfn5122f_deregister; 663 queue->q.f.ctrl = sfn5122f_control; 664 queue->q.f.notify = sfn5122f_notify; 665 queue->q.f.destroy = sfn5122f_destroy; 666 667 *q = queue; 668 669 queues[queue->id] = queue; 670 671 return SYS_ERR_OK; 672} 673 674void sfn5122f_queue_get_netfilter_ep(struct sfn5122f_queue* q, struct capref* ep) 675{ 676 *ep = q->filter_ep; 677} 678 679uint64_t sfn5122f_queue_get_id(struct sfn5122f_queue* q){ 680 return q->id; 681} 682 683struct bench_ctl* sfn5122f_get_benchmark_data(struct devq* q, uint8_t type) 684{ 685 return sfn5122f_queue_get_benchmark_data((struct sfn5122f_queue*) q, type); 686} 687 688