1/* 2 * Copyright (c) 2014 ETH Zurich. 3 * All rights reserved. 4 * 5 * This file is distributed under the terms in the attached LICENSE file. 6 * If you do not find this file, copies can be found by writing to: 7 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 8 */ 9 10#include <stdio.h> 11#include <sys/param.h> 12 13#include <barrelfish/barrelfish.h> 14#include <bulk_transfer/bulk_transfer.h> 15 16#include "bulk_net_backend.h" 17 18#define L234HEADER_SIZE (14 + 20 + 16) 19#define NDESCS 1024 20 21//#define DEBUG(x...) debug_printf(x) 22#define DEBUG(x...) do {} while (0) 23 24 25/******************************************************************************/ 26/* Protocol used on the wire */ 27 28/** Message types */ 29enum proto_msg { 30 PROTO_INVALID = 0, 31 PROTO_BIND_REQUEST, 32 PROTO_BIND_RESPONSE, 33 PROTO_DATA_TRANSFER, 34}; 35 36/** Bind request */ 37struct proto_trail_bind_req { 38 uint32_t buffer_size; 39 uint32_t meta_size; 40 41 uint8_t type; 42} __attribute__((packed)); 43 44/** Bind response */ 45struct proto_trail_bind_resp { 46 errval_t err; 47 48 uint8_t type; 49} __attribute__((packed)); 50 51/** Data transfer */ 52struct proto_trail_data_transfer { 53 uint8_t type; 54} __attribute__((packed)); 55 56 57 58/*errval_t (*bind_received)(struct bulk_channel *channel); 59void (*teardown_received)(struct bulk_channel *channel); 60errval_t (*pool_removed)(struct bulk_channel *channel, 61 struct bulk_pool *pool); 62void (*copy_released)(struct bulk_channel *channel, 63 struct bulk_buffer *buffer);*/ 64 65static errval_t cb_pool_assigned(struct bulk_channel *channel, 66 struct bulk_pool *pool); 67static void cb_move_received(struct bulk_channel *channel, 68 struct bulk_buffer *buffer, 69 void *meta); 70static void cb_buffer_received(struct bulk_channel *channel, 71 struct bulk_buffer *buffer, 72 void *meta); 73static void cb_copy_received(struct bulk_channel *channel, 74 struct bulk_buffer *buffer, 75 void *meta); 76 77static struct bulk_channel_callbacks callbacks = { 78 .pool_assigned = cb_pool_assigned, 79 .move_received = cb_move_received, 80 .buffer_received = cb_buffer_received, 81 .copy_received = cb_copy_received, 82}; 83 84 85/** Adapt MAC/IP/Port combinations in all transmit buffers (header part) */ 86static void update_tx_headers(struct bulk_net_proxy *p) 87{ 88 size_t i; 89 struct packet_header *hdr; 90 DEBUG("Updating TX headers %"PRIx64" %"PRIx64" frst=%"PRIx64"\n", 91 p->r_mac, p->l_mac, p->tb[NDESCS-2].hdr_phys); 92 for (i = 0; i < NDESCS - 1; i++) { 93 hdr = p->tb[i].hdr_virt; 94 memset(hdr, 0, sizeof(*hdr)); 95 memcpy(hdr->l2.dmac, &p->r_mac, 6); 96 memcpy(hdr->l2.smac, &p->l_mac, 6); 97 hdr->l2.type = htons(0x0800); 98 99 hdr->l3.ver_ihl = 5 | (4 << 4); 100 hdr->l3.ttl = 64; 101 hdr->l3.proto = 0x11; 102 hdr->l3.s_ip = htonl(p->l_ip); 103 hdr->l3.d_ip = htonl(p->r_ip); 104 105 hdr->l4.s_port = htons(p->l_port); 106 hdr->l4.d_port = htons(p->r_port); 107 } 108} 109 110static void add_header(struct bulk_net_msgdesc *msg) 111{ 112 struct transmit_buffer *tb = msg->parts[1].opaque; 113 struct packet_header *h = tb->hdr_virt; 114 size_t i; 115 size_t len = 0; 116 117 for (i = 1; i < BULK_NET_DESCLEN && msg->parts[i].size != 0; i++) { 118 len += msg->parts[i].size; 119 } 120 121 msg->parts[0].phys = tb->hdr_phys; 122 msg->parts[0].size = sizeof(*h); 123 msg->parts[0].opaque = NULL; 124 125 h->l4.len = htons(len + 8); 126 h->l3.len = htons(len + 8 + 20); 127} 128 129static void strip_padding(struct bulk_net_msgdesc *msg) 130{ 131 struct receive_buffer *rb = msg->parts[0].opaque; 132 struct packet_header *h = rb->hdr_virt; 133 size_t len = ntohs(h->l4.len) - 8; 134 size_t i; 135 136 for (i = 1; i < BULK_NET_DESCLEN && msg->parts[i].size != 0; i++) { 137 msg->parts[i].size = MIN(msg->parts[i].size, len); 138 len -= msg->parts[i].size; 139 } 140} 141 142static void dump_rx_msg(struct bulk_net_msgdesc *msg) 143{ 144 size_t i, j; 145 uint8_t *data; 146 uintptr_t phys; 147 struct receive_buffer *rb; 148 149#if !DO_MSG_DUMP 150 return; 151#endif 152 153 DEBUG("dump_rx_msg():\n"); 154 for (i = 0; i < BULK_NET_DESCLEN && msg->parts[i].size != 0; i++) { 155 rb = msg->parts[i].opaque; 156 DEBUG(" parts[%"PRId64"]: size=%"PRIx64" op=%p ", 157 i, msg->parts[i].size, rb); 158 if (i == 0) { 159 data = rb->hdr_virt; 160 phys = rb->hdr_phys; 161 } else { 162 data = rb->virt; 163 phys = rb->phys; 164 } 165 printf(" phys=%"PRIx64" virt=%p ", phys, data); 166 for (j = 0; j < msg->parts[i].size; j++) { 167 printf("%02"PRIx8" ", data[j]); 168 } 169 printf("\n"); 170 } 171} 172 173static void dump_tx_msg(struct bulk_net_msgdesc *msg) 174{ 175#if !DO_MSG_DUMP 176 return; 177#endif 178 size_t i, j; 179 uint8_t *data; 180 struct transmit_buffer *tb; 181 182 DEBUG("dump_tx_msg():\n"); 183 for (i = 0; i < BULK_NET_DESCLEN && msg->parts[i].size != 0; i++) { 184 DEBUG(" parts[%"PRId64"]: size=%"PRIx64" ", i, 185 msg->parts[i].size); 186 tb = msg->parts[i].opaque; 187 if (i == 0) { 188 tb = msg->parts[1].opaque; 189 data = tb->hdr_virt; 190 } else if (tb->buffer == NULL) { 191 data = tb->int_virt; 192 } else { 193 data = tb->buffer->address; 194 } 195 for (j = 0; j < msg->parts[i].size; j++) { 196 printf("%02"PRIx8" ", data[j]); 197 } 198 printf("\n"); 199 } 200} 201 202/******************************************************************************/ 203/* Sending messages to other end */ 204 205/** Send out bind request */ 206static void send_bind_request(struct bulk_net_proxy *p, 207 size_t buffer_size, 208 size_t meta_size) 209{ 210 errval_t err; 211 struct proto_trail_bind_req *t; 212 struct transmit_buffer *tb; 213 struct bulk_net_msgdesc msg; 214 215 tb = stack_alloc_alloc(&p->tb_stack); 216 assert(tb != NULL); 217 218 t = tb->int_virt; 219 t->buffer_size = buffer_size; 220 t->meta_size = meta_size; 221 t->type = PROTO_BIND_REQUEST; 222 223 msg.parts[1].phys = tb->int_phys; 224 msg.parts[1].size = sizeof(*t); 225 msg.parts[1].opaque = tb; 226 msg.parts[2].size = 0; 227 228 add_header(&msg); 229 err = bulk_e10k_send(&p->transfer, &msg); 230 assert(err_is_ok(err)); 231} 232 233/** Send out bind response */ 234static void send_bind_response(struct bulk_net_proxy *p, 235 errval_t err) 236{ 237 struct proto_trail_bind_resp *t; 238 struct transmit_buffer *tb; 239 struct bulk_net_msgdesc msg; 240 241 tb = stack_alloc_alloc(&p->tb_stack); 242 assert(tb != NULL); 243 244 t = tb->int_virt; 245 t->err = err; 246 t->type = PROTO_BIND_RESPONSE; 247 248 msg.parts[1].phys = tb->int_phys; 249 msg.parts[1].size = sizeof(*t); 250 msg.parts[1].opaque = tb; 251 msg.parts[2].size = 0; 252 253 add_header(&msg); 254 err = bulk_e10k_send(&p->transfer, &msg); 255 assert(err_is_ok(err)); 256} 257 258/** Send data transfer */ 259static void send_data_transfer(struct bulk_net_proxy *p, 260 struct bulk_buffer *b, 261 void *meta, 262 bool is_copy) 263{ 264 DEBUG("send_data_transfer()\n"); 265 errval_t err; 266 struct proto_trail_data_transfer *t; 267 struct transmit_buffer *tb_d, *tb; 268 struct bulk_net_msgdesc msg; 269 270 tb_d = stack_alloc_alloc(&p->tb_stack); 271 assert(tb_d != NULL); 272 tb_d->buffer = b; 273 tb_d->is_copy = is_copy; 274 275 // prepare trailer 276 tb = stack_alloc_alloc(&p->tb_stack); 277 assert(tb != NULL); 278 279 memcpy(tb->int_virt, meta, p->channel.meta_size); 280 t = (void *) ((uint8_t *) tb->int_virt + p->channel.meta_size); 281 t->type = PROTO_DATA_TRANSFER; 282 283 msg.parts[1].phys = b->phys; 284 msg.parts[1].size = p->buffer_size; 285 msg.parts[1].opaque = tb_d; 286 msg.parts[2].phys = tb->int_phys; 287 msg.parts[2].size = sizeof(*t) + p->channel.meta_size; 288 msg.parts[2].opaque = tb; 289 msg.parts[3].size = 0; 290 291 add_header(&msg); 292 dump_tx_msg(&msg); 293 err = bulk_e10k_send(&p->transfer, &msg); 294 assert(err_is_ok(err)); 295 DEBUG("sent_data_transfer()\n"); 296} 297 298 299/******************************************************************************/ 300/* Receiving messages from other end */ 301 302static void free_rb(struct bulk_net_proxy *p, 303 struct receive_buffer *rb) 304{ 305 if (rb->buffer == NULL) { 306 // Temporary initialization buffer -> do not reenqueue after 307 // initialization is done 308 if (p->net_bound) { 309 // TODO: free, currently leaking here 310 stack_alloc_free(&p->rb_stack, rb); 311 return; 312 } 313 } 314 315 bulk_e10k_rx_add(&p->transfer, rb->phys, rb->hdr_phys, rb); 316} 317 318static void free_rx(struct bulk_net_proxy *p, 319 struct bulk_net_msgdesc *msg) 320{ 321 size_t i; 322 323 for (i = 1; i < BULK_NET_DESCLEN && msg->parts[i].size != 0; i++) { 324 free_rb(p, msg->parts[i].opaque); 325 } 326} 327 328/** Handle received bind request */ 329static void bind_req_received(struct bulk_net_proxy *p, 330 struct proto_trail_bind_req *t, 331 struct bulk_net_msgdesc *msg) 332{ 333 struct receive_buffer *rb = msg->parts[0].opaque; 334 struct packet_header *hdr = rb->hdr_virt; 335 336 if (p->net_bound) { 337 DEBUG("Ignoring bind request to already bound proxy\n"); 338 goto free; 339 } 340 341 p->r_mac = 0; 342 memcpy(&p->r_mac, hdr->l2.smac, 6); 343 p->r_ip = ntohl(hdr->l3.s_ip); 344 p->r_port = ntohs(hdr->l4.s_port); 345 346 update_tx_headers(p); 347 348 assert(t->buffer_size == p->buffer_size); 349 send_bind_response(p, SYS_ERR_OK); 350 p->net_bound = true; 351 p->connected(p); 352 353free: 354 free_rx(p, msg); 355} 356 357/** Handle received bind response */ 358static void bind_resp_received(struct bulk_net_proxy *p, 359 struct proto_trail_bind_resp *t, 360 struct bulk_net_msgdesc *msg) 361{ 362 if (p->net_bound) { 363 DEBUG("Ignoring bind response to already bound proxy\n"); 364 goto free; 365 } 366 367 if (err_is_ok(t->err)) { 368 p->net_bound = true; 369 p->connected(p); 370 } else { 371 USER_PANIC("Remote bind attempt failed\n"); 372 } 373 374free: 375 free_rx(p, msg); 376} 377 378/** Handle received data transfer */ 379static void data_transfer_received(struct bulk_net_proxy *p, 380 struct proto_trail_data_transfer *t, 381 struct bulk_net_msgdesc *msg) 382{ 383 errval_t err; 384 struct receive_buffer *rb; 385 struct bulk_buffer *buffer; 386 387 assert(msg->parts[1].size == p->buffer_size); 388 // TODO: assumes that meta_size has a reasonably small size 389 assert(msg->parts[2].size == p->channel.meta_size + sizeof(*t)); 390 assert(msg->parts[3].size == 0); 391 392 rb = msg->parts[1].opaque; 393 buffer = rb->buffer; 394 stack_alloc_free(&p->rb_stack, rb); 395 396 rb = msg->parts[2].opaque; 397 398 err = bulk_channel_move(&p->channel, buffer, rb->virt, p->panic_cont); 399 assert(err_is_ok(err)); 400 401 free_rb(p, rb); 402} 403 404static void tcb_received(struct bulk_e10k* bu, struct bulk_net_msgdesc *msg) 405{ 406 struct bulk_net_proxy *p = bu->opaque; 407 size_t i; 408 struct receive_buffer *rb; 409 uint8_t *t; 410 DEBUG("tcb_received()\n"); 411 412 assert(msg->parts[0].size == sizeof(struct packet_header)); 413 dump_rx_msg(msg); 414 strip_padding(msg); 415 dump_rx_msg(msg); 416 417 for (i = 0; i < BULK_NET_DESCLEN && msg->parts[i].size != 0; i++); 418 i--; 419 420 rb = msg->parts[i].opaque; 421 t = rb->virt; 422 switch (t[msg->parts[i].size - 1]) { 423 case PROTO_BIND_REQUEST: 424 DEBUG("Received bind request\n"); 425 bind_req_received(p, (struct proto_trail_bind_req *) (t + 426 msg->parts[i].size - sizeof(struct proto_trail_bind_req)), 427 msg); 428 break; 429 430 case PROTO_BIND_RESPONSE: 431 DEBUG("Received bind response\n"); 432 bind_resp_received(p, (struct proto_trail_bind_resp *) (t + 433 msg->parts[i].size - sizeof(struct proto_trail_bind_resp)), 434 msg); 435 break; 436 437 case PROTO_DATA_TRANSFER: 438 DEBUG("Received data transfer\n"); 439 data_transfer_received(p, (struct proto_trail_data_transfer *) (t + 440 msg->parts[i].size - 441 sizeof(struct proto_trail_data_transfer)), msg); 442 break; 443 444 default: 445 USER_PANIC("Unexpected message type received\n"); 446 } 447} 448 449 450/******************************************************************************/ 451/* Management of network channel */ 452 453static void tcb_transmitted(struct bulk_e10k *bu, void *opaque) 454{ 455 struct bulk_net_proxy *p = bu->opaque; 456 struct transmit_buffer *tb = opaque; 457 errval_t err; 458 DEBUG("tcb_transmitted()\n"); 459 460 if (opaque == NULL) { 461 // We can ignore the header buffers 462 return; 463 } 464 465 // If there is a bulk buffer attached, need to pass it back 466 if (tb->buffer != NULL) { 467 if (tb->is_copy) { 468 err = bulk_channel_release(&p->channel, tb->buffer, p->panic_cont); 469 } else { 470 err = bulk_channel_pass(&p->channel, tb->buffer, p->zero_meta, 471 p->panic_cont); 472 } 473 assert(err_is_ok(err)); 474 tb->buffer = NULL; 475 } 476 stack_alloc_free(&p->tb_stack, tb); 477} 478 479static errval_t t_init(struct bulk_net_proxy *p) 480{ 481 errval_t err; 482 size_t i; 483 size_t n = NDESCS - 1; 484 struct receive_buffer *rb; 485 struct transmit_buffer *tb; 486 void *h_vbase, *i_vbase; 487 uintptr_t h_pbase, i_pbase; 488 489 p->net_bound = false; 490 p->transfer.opaque = p; 491 492 err = bulk_e10k_init(&p->transfer, p->ws, p->card, p->queue, p->buffer_size, 493 NDESCS, tcb_received, tcb_transmitted); 494 495 stack_alloc_init(&p->rb_stack, n); 496 stack_alloc_init(&p->tb_stack, n); 497 rb = calloc(n, sizeof(*rb)); 498 p->tb = tb = calloc(n, sizeof(*tb)); 499 500 err = allocmap_frame(E10K_HDRSZ * n * 2, &h_vbase, &h_pbase, NULL); 501 assert(err_is_ok(err)); 502 err = allocmap_frame(INT_BUFSZ * n, &i_vbase, &i_pbase, NULL); 503 assert(err_is_ok(err)); 504 505 for (i = 0; i < n; i++) { 506 rb[i].hdr_virt = h_vbase; 507 rb[i].hdr_phys = h_pbase; 508 h_pbase += E10K_HDRSZ; 509 h_vbase = (void *) ((uintptr_t) h_vbase + E10K_HDRSZ); 510 511 tb[i].hdr_virt = h_vbase; 512 tb[i].hdr_phys = h_pbase; 513 tb[i].int_virt = i_vbase; 514 tb[i].int_phys = i_pbase; 515 h_pbase += E10K_HDRSZ; 516 h_vbase = (void *) ((uintptr_t) h_vbase + E10K_HDRSZ); 517 i_pbase += INT_BUFSZ; 518 i_vbase = (void *) ((uintptr_t) i_vbase + INT_BUFSZ); 519 520 stack_alloc_free(&p->rb_stack, rb + i); 521 stack_alloc_free(&p->tb_stack, tb + i); 522 } 523 524 rb = stack_alloc_alloc(&p->rb_stack); 525 rb->buffer = NULL; 526 err = allocmap_frame(p->buffer_size, &rb->virt, &rb->phys, NULL); 527 assert(err_is_ok(err)); 528 529 err = bulk_e10k_rx_add(&p->transfer, rb->phys, rb->hdr_phys, rb); 530 assert(err_is_ok(err)); 531 532 p->l_mac = p->transfer.mac; 533 bulk_e10k_ip_info(&p->transfer, &p->l_ip); 534 return err; 535} 536 537static errval_t t_export(struct bulk_net_proxy *p) 538{ 539 errval_t err; 540 541 err = t_init(p); 542 if (err_is_fail(err)) { 543 return err; 544 } 545 546 err = bulk_e10k_port_add(&p->transfer, p->l_port); 547 if (err_is_fail(err)) { 548 return err; 549 } 550 551 return err; 552} 553 554static errval_t t_bind(struct bulk_net_proxy *p) 555{ 556 errval_t err; 557 558 err = t_init(p); 559 if (err_is_fail(err)) { 560 return err; 561 } 562 563 err = bulk_e10k_arp_lookup(&p->transfer, p->r_ip, &p->r_mac); 564 if (err_is_fail(err)) { 565 return err; 566 } 567 568 err = bulk_e10k_port_alloc(&p->transfer, &p->l_port); 569 if (err_is_fail(err)) { 570 return err; 571 } 572 573 update_tx_headers(p); 574 return err; 575} 576 577/******************************************************************************/ 578/* Bulk transfer callbacks */ 579 580static errval_t cb_pool_assigned(struct bulk_channel *channel, 581 struct bulk_pool *pool) 582{ 583 struct bulk_net_proxy *p = channel->user_state; 584 assert(pool->buffer_size == p->buffer_size); 585 return SYS_ERR_OK; 586} 587 588static void cb_move_received(struct bulk_channel *channel, 589 struct bulk_buffer *buffer, 590 void *meta) 591{ 592 DEBUG("cb_move_received()\n"); 593 struct bulk_net_proxy *p = channel->user_state; 594 assert(p->bulk_bound && p->net_bound); 595 send_data_transfer(p, buffer, meta, false); 596} 597 598static void cb_buffer_received(struct bulk_channel *channel, 599 struct bulk_buffer *buffer, 600 void *meta) 601{ 602 DEBUG("cb_buffer_received(b=%p,b->p=%"PRIx64")\n", buffer, 603 buffer->phys); 604 errval_t err; 605 struct bulk_net_proxy *p = channel->user_state; 606 struct receive_buffer *rb; 607 assert(p->bulk_bound && p->net_bound); 608 609 rb = stack_alloc_alloc(&p->rb_stack); 610 assert(rb != NULL); 611 612 rb->virt = buffer->address; 613 rb->phys = buffer->phys; 614 rb->buffer = buffer; 615 616 err = bulk_e10k_rx_add(&p->transfer, rb->phys, rb->hdr_phys, rb); 617 assert(err_is_ok(err)); 618 DEBUG("added buffer to rx queue\n"); 619} 620 621static void cb_copy_received(struct bulk_channel *channel, 622 struct bulk_buffer *buffer, 623 void *meta) 624{ 625 DEBUG("cb_copy_received()\n"); 626 struct bulk_net_proxy *p = channel->user_state; 627 assert(p->bulk_bound && p->net_bound); 628 send_data_transfer(p, buffer, meta, true); 629} 630 631 632/******************************************************************************/ 633/* Initialization */ 634 635static void cb_bind(void *arg, errval_t err, struct bulk_channel *c) 636{ 637 struct bulk_net_proxy *p = arg; 638 p->err = err; 639 p->bulk_bound = true; 640} 641 642static errval_t channel_bind(struct bulk_net_proxy *p, 643 struct bulk_endpoint_descriptor *epd) 644{ 645 errval_t err; 646 struct bulk_channel_bind_params bind_params = { 647 .role = BULK_ROLE_SLAVE, 648 .trust = BULK_TRUST_FULL, 649 .waitset = p->ws, 650 }; 651 struct bulk_continuation cont = { 652 .handler = cb_bind, 653 .arg = p, 654 }; 655 DEBUG("before bulk_channel_bind, %p\n", epd->f->channel_bind); 656 657 658 p->bulk_bound = false; 659 err = bulk_channel_bind(&p->channel, epd, &callbacks, &bind_params, cont); 660 if (err_is_fail(err)) { 661 return err; 662 } 663 664 p->channel.user_state = p; 665 666 while (!p->bulk_bound) { 667 event_dispatch(p->ws); 668 } 669 670 p->zero_meta = calloc(1, p->channel.meta_size); 671 672 return p->err; 673} 674 675errval_t bulk_net_proxy_listen(struct bulk_net_proxy *p, 676 struct bulk_endpoint_descriptor *desc, 677 struct waitset *ws, 678 size_t buffer_size, 679 const char *card, 680 uint8_t queue, 681 uint16_t port, 682 void (*connected)(struct bulk_net_proxy *)) 683{ 684 errval_t err; 685 p->card = card; 686 p->queue = queue; 687 p->ws = ws; 688 p->buffer_size = buffer_size; 689 p->l_port = port; 690 p->connected = connected; 691 692 err = channel_bind(p, desc); 693 if (err_is_fail(err)) { 694 return err; 695 } 696 697 return t_export(p); 698} 699 700errval_t bulk_net_proxy_connect(struct bulk_net_proxy *p, 701 struct bulk_endpoint_descriptor *desc, 702 struct waitset *ws, 703 size_t buffer_size, 704 const char *card, 705 uint8_t queue, 706 uint32_t ip, 707 uint16_t port, 708 void (*connected)(struct bulk_net_proxy *)) 709{ 710 errval_t err; 711 DEBUG("inside proxy connect, %p\n", ws); 712 p->ws = ws; 713 p->card = card; 714 p->queue = queue; 715 p->r_port = port; 716 p->buffer_size = buffer_size; 717 p->r_ip = ip; 718 p->connected = connected; 719 720 DEBUG("before channel bind. %p, %p, %p\n", p, desc, desc->f->channel_bind); 721 err = channel_bind(p, desc); 722 if (err_is_fail(err)) { 723 return err; 724 } 725 726 DEBUG("before tbind\n"); 727 err = t_bind(p); 728 if (err_is_fail(err)) { 729 return err; 730 } 731 732 DEBUG("Sending bind request...\n"); 733 send_bind_request(p, p->buffer_size, p->channel.meta_size); 734 DEBUG("Sent bind request\n"); 735 return SYS_ERR_OK; 736} 737