1/** 2 * \file 3 * \brief Multi-hop latency test 4 */ 5 6/* 7 * Copyright (c) 2009, 2010, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <stdio.h> 16#include <barrelfish/barrelfish.h> 17#include <string.h> 18#include <barrelfish/nameservice_client.h> 19#include <barrelfish/spawn_client.h> 20#include <bench/bench.h> 21#include <if/bench_defs.h> 22#include <unistd.h> 23#include <trace/trace.h> 24#include <trace_definitions/trace_defs.h> 25 26static char my_name[100]; 27 28// the iref of the server 29static iref_t iref; 30 31// the binding we use for the benchmark 32static struct bench_binding *binding; 33 34// waitset used for the benchmark 35struct waitset signal_waitset; 36 37// the binding we use for signaling 38static struct bench_binding *signaling_binding; 39 40static coreid_t my_core_id; 41 42// number of iterations 43#define MAX_COUNT 1000 44 45// buffers to send 46static uint8_t buffer; 47static uint8_t buffer2[100]; 48static uint8_t buffer3[1000]; 49 50// a time stamp 51struct timestamp { 52 cycles_t time0; 53 cycles_t time1; 54}; 55 56// array where we store all measured times 57static struct timestamp timestamps[MAX_COUNT]; 58 59// variables to determine when all channels are free 60static bool reply_received = false; 61static bool signal_received = false; 62 63// loop variable 64static int i = 0; 65 66// get message name 67inline static char* get_message_name(int message_type) 68{ 69 switch (message_type) { 70 case 0: 71 return "empty"; 72 case 1: 73 return "payload32_1"; 74 case 2: 75 return "payload32_2"; 76 case 3: 77 return "payload32_4"; 78 case 4: 79 return "payload32_8"; 80 case 5: 81 return "payload32_16"; 82 case 6: 83 return "payload64_1"; 84 case 7: 85 return "payload64_2"; 86 case 8: 87 return "payload64_4"; 88 case 9: 89 return "payload64_8"; 90 case 10: 91 return "payload_64_16"; 92 case 11: 93 return "buffer (1 byte)"; 94 case 12: 95 return "buffer (100 bytes)"; 96 case 13: 97 return "buffer (1000 bytes)"; 98 99 default: 100 printf("unknown message type\n"); 101 abort(); 102 return ""; 103 } 104} 105 106static void experiment_cont(void* arg); 107 108inline static void experiment(void) 109{ 110 111 if (!(reply_received && signal_received)) { 112 return; 113 }assert(reply_received && signal_received); 114 115 reply_received = false; 116 signal_received = false; 117 118 // continue experiment as soon as binding can accept 119 // the next message 120 errval_t err; 121 err = binding->register_send(binding, get_default_waitset(), 122 MKCONT(experiment_cont, NULL)); 123 if (err_is_fail(err)) { 124 USER_PANIC_ERR(err, "error in register send"); 125 } 126} 127 128// continue experiment 129static void experiment_cont(void* arg) 130{ 131 132 errval_t err; 133 static bool flag = false; 134 static int message_type = 0; 135 136 // Experiment finished (with this message type) 137 if (i == MAX_COUNT - 1) { 138 139#if CONFIG_TRACE 140#else 141 // print measured times 142 for (int j = MAX_COUNT / 10; j < MAX_COUNT; j++) { 143 144 printf( 145 "page %d took %"PRIuCYCLES"\n", 146 j, 147 timestamps[j].time1 - bench_tscoverhead() 148 - timestamps[j].time0); 149 } 150#endif 151 // go to next message type 152 message_type++; 153 flag = false; 154 i = 0; 155 if (message_type > 13) { 156 157 // stop tracing 158 err = trace_event(TRACE_SUBSYS_MULTIHOP, 159 TRACE_EVENT_MULTIHOP_BENCH_STOP, 0); 160 if (err_is_fail(err)) { 161 USER_PANIC_ERR(err, "trace_event failed"); 162 } 163 164#if CONFIG_TRACE 165 // dump trace 166 char *buf = malloc(50*4096*4096); 167 size_t length = trace_dump(buf, 20*4096*4096, NULL); 168 printf("%s\n", buf); 169 printf("length of buffer %lu\n", length); 170#endif 171 printf("client done!\n"); 172 return; 173 } 174 } 175 176 if (!flag) { // Start experiment 177 178#if CONFIG_TRACE 179#else 180 printf("Running latency test for message %s...\n", 181 get_message_name(message_type)); 182#endif 183 flag = true; 184 timestamps[i].time0 = bench_tsc(); 185 } else { // Continue experiment 186 i++; 187 timestamps[i].time0 = bench_tsc(); 188 } 189 190 // trace send event 191 err = trace_event(TRACE_SUBSYS_MULTIHOP, TRACE_EVENT_MULTIHOP_MESSAGE_SEND, 192 message_type); 193 if (err_is_fail(err)) { 194 USER_PANIC_ERR(err, "trace_event failed"); 195 } 196 197 // send next message 198 switch (message_type) { 199 case 0: 200 err = binding->tx_vtbl.fsb_empty_request(binding, NOP_CONT); 201 break; 202 case 1: 203 err = binding->tx_vtbl.fsb_payload32_1_request(binding, NOP_CONT, 1); 204 break; 205 206 case 2: 207 err = binding->tx_vtbl.fsb_payload32_2_request(binding, NOP_CONT, 1, 2); 208 break; 209 210 case 3: 211 err = binding->tx_vtbl.fsb_payload32_4_request(binding, NOP_CONT, 1, 2, 212 3, 4); 213 break; 214 215 case 4: 216 err = binding->tx_vtbl.fsb_payload32_8_request(binding, NOP_CONT, 1, 2, 217 3, 4, 5, 6, 7, 8); 218 break; 219 220 case 5: 221 err = binding->tx_vtbl.fsb_payload32_16_request(binding, NOP_CONT, 1, 2, 222 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); 223 break; 224 225 case 6: 226 err = binding->tx_vtbl.fsb_payload64_1_request(binding, NOP_CONT, 1); 227 break; 228 229 case 7: 230 err = binding->tx_vtbl.fsb_payload64_2_request(binding, NOP_CONT, 1, 2); 231 break; 232 233 case 8: 234 err = binding->tx_vtbl.fsb_payload64_4_request(binding, NOP_CONT, 1, 2, 235 3, 4); 236 break; 237 238 case 9: 239 err = binding->tx_vtbl.fsb_payload64_8_request(binding, NOP_CONT, 1, 2, 240 3, 4, 5, 6, 7, 8); 241 break; 242 243 case 10: 244 err = binding->tx_vtbl.fsb_payload64_16_request(binding, NOP_CONT, 1, 2, 245 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); 246 break; 247 248 case 11: 249 err = binding->tx_vtbl.fsb_buffer_request(binding, NOP_CONT, &buffer, 250 1); 251 break; 252 253 case 12: 254 err = binding->tx_vtbl.fsb_buffer_request(binding, NOP_CONT, buffer2, 255 100); 256 break; 257 258 case 13: 259 err = binding->tx_vtbl.fsb_buffer_request(binding, NOP_CONT, buffer3, 260 1000); 261 break; 262 263 default: 264 printf("unknown message type\n"); 265 abort(); 266 break; 267 } 268 269 // make sure send was successful 270 if (err_is_fail(err)) { 271 USER_PANIC_ERR(err, "while running experiment\n"); 272 } 273 274 // receive reply (by dispatching events from the 275 // waitset we use for the benchmark) 276 while (reply_received == false) { 277 event_dispatch(&signal_waitset); 278 } 279 280 experiment(); 281} 282 283// called when we receive a signal 284static void busy_ping(struct bench_binding *b) 285{ 286 signal_received = true; 287 experiment(); 288} 289 290// called when a message is received 291static inline void message_received(void) 292{ 293 errval_t err; 294 295 // save timestamp 296 timestamps[i].time1 = bench_tsc(); 297 298 // trace receive event 299 err = trace_event(TRACE_SUBSYS_MULTIHOP, 300 TRACE_EVENT_MULTIHOP_MESSAGE_RECEIVE, 0); 301 if (err_is_fail(err)) { 302 USER_PANIC_ERR(err, "trace_event failed"); 303 } 304 305 reply_received = true; 306} 307 308// send continue signal to server 309static void continue_signal(void* arg) 310{ 311 errval_t err; 312 err = signaling_binding->tx_vtbl.busy_ping(signaling_binding, NOP_CONT); 313 assert(err_is_ok(err)); 314} 315 316static void fsb_init_msg(struct bench_binding *b, coreid_t id) 317{ 318 errval_t err; 319 320 // change waitset of the binding 321 waitset_init(&signal_waitset); 322 err = b->change_waitset(b, &signal_waitset); 323 assert(err_is_ok(err)); 324 325 binding = b; 326 reply_received = true; 327 328#if CONFIG_TRACE 329 // configure tracing 330 err = trace_control(TRACE_EVENT(TRACE_SUBSYS_MULTIHOP, 331 TRACE_EVENT_MULTIHOP_BENCH_START, 0), 332 TRACE_EVENT(TRACE_SUBSYS_MULTIHOP, 333 TRACE_EVENT_MULTIHOP_BENCH_STOP, 0), 0); 334 if(err_is_fail(err)) { 335 USER_PANIC_ERR(err, "trace_control failed"); 336 } 337#endif 338 339 // start tracing 340 err = trace_event(TRACE_SUBSYS_MULTIHOP, TRACE_EVENT_MULTIHOP_BENCH_START, 341 0); 342 if (err_is_fail(err)) { 343 USER_PANIC_ERR(err, "trace_event failed"); 344 } 345 346 experiment(); 347} 348 349static void fsb_empty_reply(struct bench_binding *b) 350{ 351 message_received(); 352} 353 354static void fsb_empty_request(struct bench_binding *b) 355{ 356 errval_t err; 357 err = b->tx_vtbl.fsb_empty_reply(b, MKCONT(continue_signal, NULL)); 358 assert(err_is_ok(err)); 359} 360 361static void fsb_payload32_1_reply(struct bench_binding *b, int32_t p0) 362{ 363 message_received(); 364} 365 366static void fsb_payload32_2_reply(struct bench_binding *b, int32_t p0, 367 int32_t p1) 368{ 369 message_received(); 370} 371 372static void fsb_payload32_4_reply(struct bench_binding *b, int32_t payload0, 373 int32_t payload1, int32_t payload2, int32_t payload3) 374{ 375 message_received(); 376} 377 378static void fsb_payload32_8_reply(struct bench_binding *b, int32_t p0, 379 int32_t p1, int32_t p2, int32_t p3, int32_t p4, int32_t p5, int32_t p6, 380 int32_t p7) 381{ 382 message_received(); 383} 384 385static void fsb_payload32_16_reply(struct bench_binding *b, int32_t p0, 386 int32_t p1, int32_t p2, int32_t p3, int32_t p4, int32_t p5, int32_t p6, 387 int32_t p7, int32_t p8, int32_t p9, int32_t p10, int32_t p11, 388 int32_t p12, int32_t p13, int32_t p14, int32_t p15) 389{ 390 message_received(); 391} 392 393static void fsb_payload32_1_request(struct bench_binding *b, int32_t payload0) 394{ 395 errval_t err; 396 err = b->tx_vtbl.fsb_payload32_1_reply(b, MKCONT(continue_signal, NULL), 1); 397 assert(err_is_ok(err)); 398} 399 400static void fsb_payload32_2_request(struct bench_binding *b, int32_t payload0, 401 int32_t payload1) 402{ 403 errval_t err; 404 err = b->tx_vtbl.fsb_payload32_2_reply(b, MKCONT(continue_signal, NULL), 1, 405 2); 406 assert(err_is_ok(err)); 407} 408 409static void fsb_payload32_4_request(struct bench_binding *b, int32_t payload0, 410 int32_t payload1, int32_t payload2, int32_t payload3) 411{ 412 errval_t err; 413 err = b->tx_vtbl.fsb_payload32_4_reply(b, MKCONT(continue_signal, NULL), 1, 414 2, 3, 4); 415 assert(err_is_ok(err)); 416} 417 418static void fsb_payload32_8_request(struct bench_binding *b, int32_t payload0, 419 int32_t payload1, int32_t payload2, int32_t payload3, int32_t payload4, 420 int32_t payload5, int32_t payload6, int32_t payload7) 421{ 422 errval_t err; 423 err = b->tx_vtbl.fsb_payload32_8_reply(b, MKCONT(continue_signal, NULL), 1, 424 2, 3, 4, 5, 6, 7, 8); 425 assert(err_is_ok(err)); 426} 427 428static void fsb_payload32_16_request(struct bench_binding *b, int32_t payload0, 429 int32_t payload1, int32_t payload2, int32_t payload3, int32_t payload4, 430 int32_t payload5, int32_t payload6, int32_t payload7, int32_t payload8, 431 int32_t payload9, int32_t payload10, int32_t payload11, 432 int32_t payload12, int32_t payload13, int32_t payload14, 433 int32_t payload15) 434{ 435 errval_t err; 436 err = b->tx_vtbl.fsb_payload32_16_reply(b, MKCONT(continue_signal, NULL), 1, 437 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); 438 assert(err_is_ok(err)); 439} 440 441static void fsb_payload64_1_reply(struct bench_binding *b, int64_t p0) 442{ 443 message_received(); 444} 445 446static void fsb_payload64_2_reply(struct bench_binding *b, int64_t p0, 447 int64_t p1) 448{ 449 message_received(); 450} 451 452static void fsb_payload64_4_reply(struct bench_binding *b, int64_t payload0, 453 int64_t payload1, int64_t payload2, int64_t payload3) 454{ 455 message_received(); 456} 457 458static void fsb_payload64_8_reply(struct bench_binding *b, int64_t p0, 459 int64_t p1, int64_t p2, int64_t p3, int64_t p4, int64_t p5, int64_t p6, 460 int64_t p7) 461{ 462 message_received(); 463} 464 465static void fsb_payload64_16_reply(struct bench_binding *b, int64_t p0, 466 int64_t p1, int64_t p2, int64_t p3, int64_t p4, int64_t p5, int64_t p6, 467 int64_t p7, int64_t p8, int64_t p9, int64_t p10, int64_t p11, 468 int64_t p12, int64_t p13, int64_t p14, int64_t p15) 469{ 470 message_received(); 471} 472 473static void fsb_payload64_1_request(struct bench_binding *b, int64_t payload0) 474{ 475 errval_t err; 476 err = b->tx_vtbl.fsb_payload64_1_reply(b, MKCONT(continue_signal, NULL), 1); 477 if (err_is_fail(err)) { 478 USER_PANIC_ERR(err, "error while sending reply message in client\n"); 479 } 480} 481 482static void fsb_payload64_2_request(struct bench_binding *b, int64_t payload0, 483 int64_t payload1) 484{ 485 errval_t err; 486 err = b->tx_vtbl.fsb_payload64_2_reply(b, MKCONT(continue_signal, NULL), 1, 487 2); 488 if (err_is_fail(err)) { 489 USER_PANIC_ERR(err, "error while sending reply message in client\n"); 490 } 491} 492 493static void fsb_payload64_4_request(struct bench_binding *b, int64_t payload0, 494 int64_t payload1, int64_t payload2, int64_t payload3) 495{ 496 errval_t err; 497 err = b->tx_vtbl.fsb_payload64_4_reply(b, MKCONT(continue_signal, NULL), 1, 498 2, 3, 4); 499 if (err_is_fail(err)) { 500 USER_PANIC_ERR(err, "error while sending reply message in client\n"); 501 } 502} 503 504static void fsb_payload64_8_request(struct bench_binding *b, int64_t payload0, 505 int64_t payload1, int64_t payload2, int64_t payload3, int64_t payload4, 506 int64_t payload5, int64_t payload6, int64_t payload7) 507{ 508 errval_t err; 509 err = b->tx_vtbl.fsb_payload64_8_reply(b, MKCONT(continue_signal, NULL), 1, 510 2, 3, 4, 5, 6, 7, 8); 511 if (err_is_fail(err)) { 512 USER_PANIC_ERR(err, "error while sending reply message in client\n"); 513 } 514} 515 516static void fsb_payload64_16_request(struct bench_binding *b, int64_t payload0, 517 int64_t payload1, int64_t payload2, int64_t payload3, int64_t payload4, 518 int64_t payload5, int64_t payload6, int64_t payload7, int64_t payload8, 519 int64_t payload9, int64_t payload10, int64_t payload11, 520 int64_t payload12, int64_t payload13, int64_t payload14, 521 int64_t payload15) 522{ 523 524 errval_t err; 525 err = b->tx_vtbl.fsb_payload64_16_reply(b, MKCONT(continue_signal, NULL), 1, 526 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); 527 if (err_is_fail(err)) { 528 USER_PANIC_ERR(err, "error while sending reply message in client\n"); 529 } 530} 531 532static void fsb_buffer_reply(struct bench_binding *b, const uint8_t *payload, 533 size_t size) 534{ 535 message_received(); 536} 537 538static void fsb_buffer_request(struct bench_binding *b, const uint8_t *payload, 539 size_t size) 540{ 541 errval_t err; 542 err = trace_event(TRACE_SUBSYS_MULTIHOP, 543 TRACE_EVENT_MULTIHOP_MESSAGE_RECEIVE, 0); 544 if (err_is_fail(err)) { 545 USER_PANIC_ERR(err, "trace_event failed"); 546 } 547 548 err = b->tx_vtbl.fsb_buffer_reply(b, MKCONT(continue_signal, NULL), payload, 549 size); 550 if (err_is_fail(err)) { 551 USER_PANIC_ERR(err, "error while sending reply message in client\n"); 552 } 553} 554 555// receive virtual table 556static struct bench_rx_vtbl rx_vtbl = { .fsb_init_msg = fsb_init_msg, 557 .fsb_empty_request = fsb_empty_request, .fsb_empty_reply = 558 fsb_empty_reply, .fsb_payload32_1_request = 559 fsb_payload32_1_request, .fsb_payload32_2_request = 560 fsb_payload32_2_request, .fsb_payload32_4_request = 561 fsb_payload32_4_request, .fsb_payload32_8_request = 562 fsb_payload32_8_request, .fsb_payload32_16_request = 563 fsb_payload32_16_request, .fsb_payload32_1_reply = 564 fsb_payload32_1_reply, .fsb_payload32_2_reply = 565 fsb_payload32_2_reply, .fsb_payload32_4_reply = 566 fsb_payload32_4_reply, .fsb_payload32_8_reply = 567 fsb_payload32_8_reply, .fsb_payload32_16_reply = 568 fsb_payload32_16_reply, .fsb_payload64_1_request = 569 fsb_payload64_1_request, .fsb_payload64_2_request = 570 fsb_payload64_2_request, .fsb_payload64_4_request = 571 fsb_payload64_4_request, .fsb_payload64_8_request = 572 fsb_payload64_8_request, .fsb_payload64_16_request = 573 fsb_payload64_16_request, .fsb_payload64_1_reply = 574 fsb_payload64_1_reply, .fsb_payload64_2_reply = 575 fsb_payload64_2_reply, .fsb_payload64_4_reply = 576 fsb_payload64_4_reply, .fsb_payload64_8_reply = 577 fsb_payload64_8_reply, .fsb_payload64_16_reply = 578 fsb_payload64_16_reply, 579 .fsb_buffer_request = fsb_buffer_request, .fsb_buffer_reply = 580 fsb_buffer_reply, .busy_ping = busy_ping, 581 582}; 583 584static void bind_cb(void *st, errval_t binderr, struct bench_binding *b) 585{ 586 // copy my message receive handler vtable to the binding 587 b->rx_vtbl = rx_vtbl; 588 589 // Send an init message. This will start the benchmark. 590 errval_t err; 591 err = b->tx_vtbl.fsb_init_msg(b, MKCONT(continue_signal, NULL), my_core_id); 592 assert(err_is_ok(err)); 593} 594 595static void bind_signal_cb(void *st, errval_t binderr, struct bench_binding *b) 596{ 597 598 errval_t err; 599 // copy my message receive handler vtable to the binding 600 b->rx_vtbl = rx_vtbl; 601 signaling_binding = b; 602 603 // bind a second time over the multi-hop interconnect driver 604 // we will use this binding for the benchmark 605 err = bench_bind(iref, bind_cb, NULL, get_default_waitset(), 606 IDC_BIND_FLAGS_DEFAULT | IDC_BIND_FLAG_MULTIHOP); 607 if (err_is_fail(err)) { 608 DEBUG_ERR(err, "bind failed"); 609 abort(); 610 } 611 612} 613 614static void export_cb(void *st, errval_t err, iref_t iref2) 615{ 616 if (err_is_fail(err)) { 617 DEBUG_ERR(err, "export failed"); 618 abort(); 619 } 620 621 // register this iref with the name service 622 err = nameservice_register("multihop_server", iref2); 623 if (err_is_fail(err)) { 624 DEBUG_ERR(err, "nameservice_register failed"); 625 abort(); 626 } 627} 628 629static errval_t connect_cb(void *st, struct bench_binding *b) 630{ 631 // copy my message receive handler vtable to the binding 632 b->rx_vtbl = rx_vtbl; 633 634 // accept the connection 635 return SYS_ERR_OK; 636} 637 638int main(int argc, char *argv[]) 639{ 640 errval_t err; 641 642 /* Set my core id */ 643 my_core_id = disp_get_core_id(); 644 strcpy(my_name, argv[0]); 645 646 printf("entered\n"); 647 bench_init(); 648 printf("bench_init done\n"); 649 650 if (argc == 1) { /* server */ 651 652 /* 653 1. spawn domain, 654 2. setup a server, 655 3. wait for client to connect, 656 4. run experiment 657 */ 658 659 char *xargv[] = { my_name, "dummy", "dummy", "dummy", NULL }; 660 err = spawn_program(1, my_name, xargv, NULL, SPAWN_FLAGS_DEFAULT, NULL); 661 assert(err_is_ok(err)); 662 663 /* Setup a server */ 664 err = bench_export(NULL, export_cb, connect_cb, get_default_waitset(), 665 IDC_BIND_FLAGS_DEFAULT); 666 assert(err_is_ok(err)); 667 668 } else { 669 /* Connect to the server */ 670 671 printf("ns lookup\n"); 672 err = nameservice_blocking_lookup("multihop_server", &iref); 673 if (err_is_fail(err)) { 674 DEBUG_ERR(err, "nameservice_blocking_lookup failed"); 675 abort(); 676 } 677 678 printf("bench_bind\n"); 679 // bind a first time for signaling 680 err = bench_bind(iref, bind_signal_cb, NULL, get_default_waitset(), 681 IDC_BIND_FLAGS_DEFAULT); 682 if (err_is_fail(err)) { 683 DEBUG_ERR(err, "bind failed"); 684 abort(); 685 } 686 } 687 messages_handler_loop(); 688 return 0; 689} 690