1/** 2 * \file 3 * \brief simple udp benchmark 4 */ 5 6/* 7 * Copyright (c) 2007-11 ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15 16#include <barrelfish/barrelfish.h> 17#include <barrelfish/waitset.h> 18#include <barrelfish/nameservice_client.h> 19#include <trace/trace.h> 20#include <trace_definitions/trace_defs.h> 21#include <stdio.h> 22#include <lwip/pbuf.h> 23#include <lwip/udp.h> 24#include <lwip/init.h> 25#include <netif/etharp.h> 26#include <netbench/netbench.h> 27 28/* Enable tracing only when it is globally enabled */ 29#if CONFIG_TRACE && NETWORK_STACK_BENCHMARK 30#define UDP_BENCHMARK_TRACE 1 31#endif // CONFIG_TRACE && NETWORK_STACK_BENCHMARK 32 33 34//#define TOTAL_DATA_SIZE 629188608 35#define MAX_DATA 1330 36#define MULTIPLIER 100 37 38//#define TEST_BUFFER_MANAGEMENT 1 39 40#ifdef TEST_BUFFER_MANAGEMENT 41#define TEST_TYPE "With BUFF Mng" 42#else 43#define TEST_TYPE "Without BUFF Mng" 44#endif // TEST_BUFFER_MANAGEMENT 45 46 47static int connection_type = 0; // 0 for using PBUF_POOL (RX path) 48//static int connection_type = 1; // 1 for PBUF_RAM (TX path) Horribly slow!! 49 50static uint64_t pkt_count = 0; 51static uint64_t rx_data_size = 0; 52static uint64_t recv_start_c = 0; 53static uint64_t recv_stop_c = 0; 54 55static uint64_t iterations = 2; 56 57static struct waitset *ws = NULL; 58 59static void 60loop_forever(void) 61{ 62 errval_t r; 63 // Loop forever 64 while (1) { 65 r = event_dispatch(ws); 66 if (err_is_fail(r)) { 67 DEBUG_ERR(r, "in event_dispatch"); 68 break; 69 } 70 } 71} 72 73static void 74refresh_cache(struct ip_addr *dst_ip) 75{ 76 struct netif *netif; 77 netif = ip_route(dst_ip); 78 79 errval_t r = etharp_request(netif, dst_ip); 80 assert(err_is_ok(r)); 81 82 while (is_ip_present_in_arp_cache(dst_ip) == false) { 83 r = event_dispatch(ws); 84 if (err_is_fail(r)) { 85 DEBUG_ERR(r, "in event_dispatch"); 86 abort(); 87 } 88 } // end while: till arp not present 89} 90 91static uint64_t stats[10] = {0, 0, 0, 0}; 92static uint64_t start_tx = 0; 93static uint64_t iter = 0; // Iteration counter 94static uint64_t failed = 0; // Failure counter 95static void stop_benchmark(uint64_t stop, uint64_t driver_runtime, 96 uint64_t drv_pkt_count) 97{ 98 // FIXME: Make sure that all data is gone 99// uint64_t stop = rdtsc(); 100 uint64_t delta = stop - start_tx; 101 102 // sending debug message marking the stop of benchmark 103// lwip_benchmark_control(connection_type, BMS_STOP_REQUEST, 0, 0); 104 105 106 printf("U: Test [%s], PBUF type %s\n", TEST_TYPE, 107 connection_type?"PBUF_RAM":"PBUF_POOL"); 108 lwip_print_interesting_stats(); 109 printf("U: Time taken by APP %"PU" to send %"PRIu64" packets" 110 "(%"PRIu64" failed)\n", 111 in_seconds(delta), iter, failed); 112 if (driver_runtime > 0) { 113 printf("U: Time taken by DRV %"PU" to send %"PRIu64" packets\n", 114 in_seconds(driver_runtime), drv_pkt_count); 115 } 116 uint64_t data_size = iter * MAX_DATA; 117 printf("U: TX speed (app view) = data(%"PRIu64") / time(%"PU") = [%f] KB \n", 118 data_size, in_seconds(delta), ((data_size/in_seconds(delta))/1024)); 119 120 if (driver_runtime > 0) { 121 data_size = drv_pkt_count * MAX_DATA; 122 printf("U: TX speed (DRV view) = data(%"PRIu64") / time(%"PU") = [%f] KB \n", 123 data_size, in_seconds(driver_runtime), 124 ((data_size/in_seconds(driver_runtime))/1024)); 125 } 126 for (int j = 0; j < 6; ++j) { 127 printf("U: Stats %d: [%"PRIu64"] \n", j, stats[j]); 128 } 129 130 loop_forever(); 131} 132 133static void wait_for_lwip(void) 134{ 135 errval_t r; 136 int ans; 137 while ((ans = is_lwip_loaded()) > 0) { 138// printf("is_lwip_loaded returned %d\n", ans); 139 ++stats[ans]; 140/* if(ans == 1) { 141 printf("stopping the benchmark as no more pbufs\n"); 142 stop_benchmark(); 143 } 144*/ 145 r = event_dispatch(ws); 146 if (err_is_fail(r)) { 147 DEBUG_ERR(r, "in event_dispatch"); 148 abort(); 149 } 150 } // end while: lwip_loaded 151 ++stats[ans]; 152} // end function: wait_for_lwip 153 154static struct pbuf * 155get_pbuf_wrapper(void) 156{ 157 struct pbuf *p = NULL; 158 if (connection_type == 1) { 159 p = pbuf_alloc(PBUF_TRANSPORT, MAX_DATA, PBUF_RAM); 160 } else { 161 p = pbuf_alloc(PBUF_TRANSPORT, MAX_DATA, PBUF_POOL); 162 // setting ref to zero as we are using it for sending and 163 // not receiving 164 } 165 if (p == NULL){ 166 printf("pbuf_alloc failed while counter %"PRIu16" \n", 167 free_pbuf_pool_count()); 168 } 169 assert(p != NULL); 170 assert(p->payload != NULL); 171 assert(p->len == p->tot_len); 172 assert(p->len == MAX_DATA); 173// memset(p->payload, 'd', p->len); 174 return p; 175 176} // end function: get_pbuf_wrapper 177 178 179static bool wait_for_driver_ready(void) 180{ 181 errval_t r; 182 uint8_t ans; 183 uint64_t delta; 184 uint64_t cl; 185 186 while (1) { 187 ans = lwip_driver_benchmark_state(connection_type, &delta, &cl); 188 if (ans == BMS_RUNNING) { 189 return true; 190 } 191 assert(ans == 1); 192 193 r = event_dispatch(ws); 194 if (err_is_fail(r)) { 195 DEBUG_ERR(r, "in event_dispatch"); 196 abort(); 197 } 198 } // end while: lwip_loaded 199 return false; 200} 201 202static bool check_for_driver_done(uint64_t *delta, uint64_t *cl) 203{ 204 uint8_t ans; 205 ans = lwip_driver_benchmark_state(connection_type, delta, cl); 206 if (ans == BMS_STOPPED) { 207 return true; 208 } 209 return false; 210} 211 212static void 213udp_sender(struct udp_pcb *upcb, struct ip_addr recv_ip, 214 uint16_t recv_port) 215{ 216 uint64_t driver_delta; 217 uint64_t cl; 218 219 struct pbuf *p = NULL; 220 printf("U: Going in UDP_SENDER mode\n"); 221 222 // connect with peer 223 errval_t r = udp_connect(upcb, &recv_ip, recv_port); 224 if (err_is_fail(r)) { 225 DEBUG_ERR(r, "udp_connect:"); 226 } 227 228#ifndef TEST_BUFFER_MANAGEMENT 229 // create a pbuf 230 printf("U: Testing without buffer manager\n"); 231 p = get_pbuf_wrapper(); 232 printf("U: pbuf len %"PRIu16", tot_len %"PRIu16"\n", 233 p->len, p->tot_len); 234 void *payload_ptr = p->payload; 235 236 // Set the data to zero 237 memset(p->payload, 'd', p->len); 238#else 239 printf("U: Testing *with* buffer manager!\n"); 240#endif // TEST_BUFFER_MANAGEMENT 241 242 refresh_cache(&recv_ip); 243 244 printf("U: Trying to send %"PRIu64" packets\n", iterations); 245 246 lwip_benchmark_control(connection_type, BMS_START_REQUEST, iterations, 0); 247 wait_for_driver_ready(); 248 start_tx = rdtsc(); 249 250 // send data 251// for (iter = 0; iter < iterations; ++iter) { 252 iter = 0; 253 while (1) { 254// wait_for_lwip(); 255 256#ifdef TEST_BUFFER_MANAGEMENT 257 p = get_pbuf_wrapper(); 258#else 259 /* resetting the values as they will be changed by 260 * pbuf_header function */ 261 p->len = MAX_DATA; 262 p->tot_len = MAX_DATA; 263 p->payload = payload_ptr; 264#endif // TEST_BUFFER_MANAGEMENT 265 266 r = udp_send(upcb, p); 267 if (err_is_fail(r)) { 268 ++failed; 269// printf("udp_send failed(%"PRIu64") for iter %"PRIu64"\n", 270// failed, iter); 271 272// DEBUG_ERR(r, "udp_send:"); 273 wait_for_lwip(); 274 } // end if: failed 275 else { 276 ++iter; 277 } 278// printf("Sent packet no. %"PRIu64"\n", i); 279 280 281#ifdef TEST_BUFFER_MANAGEMENT 282 pbuf_free(p); 283#endif // TEST_BUFFER_MANAGEMENT 284 285 if (iter == (iterations)) { 286 driver_delta = 0; 287 break; 288 } 289 290 if (check_for_driver_done(&driver_delta, &cl) == true) { 291 break; 292 } 293 294 } // end while : 295 296 lwip_benchmark_control(connection_type, BMS_STOP_REQUEST, 0, 0); 297 298 while (check_for_driver_done(&driver_delta, &cl) == false) { 299 r = event_dispatch(ws); 300 if (err_is_fail(r)) { 301 DEBUG_ERR(r, "in event_dispatch"); 302 break; 303 } 304 } 305 306 uint64_t stop_tx = rdtsc(); 307 stop_benchmark(stop_tx, driver_delta, cl); 308 wait_for_lwip(); 309} // end function: udp_sender 310 311 312 313// ################################################ receiver benchmark #### 314 315static bool udp_recv_bm_shown = false; 316static void 317udp_receiver_done(void) 318{ 319 // Record the stop timer 320 recv_stop_c = rdtsc(); 321 uint64_t delta = recv_stop_c - recv_start_c; 322 lwip_benchmark_control(connection_type, BMS_STOP_REQUEST, 0, 0); 323 324 lwip_print_interesting_stats(); 325 // print the statistics 326 printf("U: Time taken %"PU" to recv %"PRIu64" data" 327 "(%"PRIu64" packets)\n", in_seconds(delta), 328 rx_data_size, pkt_count); 329 printf("U: RX speed = data(%"PRIu64") / time(%"PU") = [%f] KB \n", 330 rx_data_size, in_seconds(delta), 331 ((rx_data_size/in_seconds(delta))/1024)); 332 udp_recv_bm_shown = true; 333} // end function: udp_receiver_done 334 335static void 336udp_recv_handler(void *arg, struct udp_pcb *pcb, struct pbuf *pbuf, 337 struct ip_addr *addr, u16_t port) 338{ 339 assert(pbuf != NULL); 340 assert(pbuf->payload != NULL); 341 assert(pbuf->tot_len > 0); 342 rx_data_size = rx_data_size + pbuf->tot_len; 343 if(pkt_count == 0){ 344 // record starting time 345 recv_start_c = rdtsc(); 346 } 347 ++pkt_count; 348 349 /* 350 if (pkt_count % 1000 == 0) { 351 printf("U: APP %"PRIu64" packets in\n", pkt_count); 352 } 353*/ 354 355#if UDP_BENCHMARK_TRACE 356 trace_event(TRACE_SUBSYS_BNET, TRACE_EVENT_BNET_APP_SEE, pkt_count); 357#endif // UDP_BENCHMARK_TRACE 358 359 if (pkt_count >= 3300) { 360// printf("APP %"PRIu64" packets in *\n", pkt_count); 361 } 362 363// if (rx_data_size >= (iterations * MAX_DATA) ) { 364 if (rx_data_size >= (1024 * 1024 * 1024) ) { 365 // condition meet 366 if (!udp_recv_bm_shown) { 367 udp_receiver_done(); 368 } 369 } 370 pbuf_free(pbuf); 371} // end function: udp_recv_handler 372 373 374static void 375udp_receiver(struct udp_pcb *upcb, struct ip_addr *listen_ip, 376 uint16_t listen_port) 377{ 378 printf("U: Going in UDP_RECEIVER mode\n"); 379 // Bind to specified port 380 errval_t r = udp_bind(upcb, listen_ip, listen_port); 381 if (err_is_fail(r)) { 382 DEBUG_ERR(r, "udp_bind:"); 383 } 384 385 lwip_benchmark_control(connection_type, BMS_START_REQUEST, 386 iterations, rdtsc()); 387 udp_recv(upcb, udp_recv_handler, 0 /*client data, arg in callback*/); 388 389 while (true) { 390 r = event_dispatch(ws); 391 if (err_is_fail(r)) { 392 DEBUG_ERR(r, "in event_dispatch"); 393 break; 394 } 395 } 396} // end function: udp_receiver 397 398 399int main(int argc, char *argv[]) 400{ 401 402 struct ip_addr peer_ip; // IP address of peer 403 uint16_t port = 0; // Port number of the peer 404 405 ws = get_default_waitset(); 406 407 // Parse args 408 if (argc != 5) { 409 printf("Usage: %s <direction> <IP> <Port> <packets * %d>\n", 410 argv[0], MULTIPLIER); 411 printf("eg (to send microbenchmark): %s 1 10.110.4.41 3000 1000\n", argv[0]); 412 printf("eg (to recv microbenchmark): %s 0 10.110.4.41 3000 1000\n", argv[0]); 413 return 1; 414 } 415 416 // Flag to choose between sender(1) and receiver(0) 417 int as_sender = atoi(argv[1]); 418 419 struct in_addr peer_ip_gen; 420 int ret = inet_aton(argv[2], &peer_ip_gen); 421 if (ret == 0) { 422 printf("Invalid IP addr: %s\n", argv[2]); 423 return 1; 424 } // end if : ip validation 425 peer_ip.addr = peer_ip_gen.s_addr; 426 427 port = atoi(argv[3]); 428 if (port <= 0) { 429 printf("Invalid port given [%s] == [%"PRIu16"]\n", 430 argv[3], port); 431 return 1; 432 } // end if : port validation 433 434 iterations = atoi(argv[4]); 435 if (iterations <= 0) { 436 printf("Invalid no. of iterations [%s] == [%"PRIu64"]\n", 437 argv[4], iterations); 438 return 1; 439 } // end if : port validation 440 iterations = iterations * MULTIPLIER; 441 442 if (lwip_init_auto() == false) { 443 printf("ERROR: lwip_init_auto failed!\n"); 444 return 1; 445 } 446 447// lwip_init("e1000"); 448 // create pcb for connection 449 struct udp_pcb *upcb; 450 upcb = udp_new(); 451 452 assert(upcb != NULL); 453 454 printf("U: #####################################\n"); 455 printf("U: %d.%"PRIuDOMAINID": Performing [%"PRIu64"] iterations\n", 456 disp_get_core_id(), disp_get_domain_id(), 457 iterations); 458 459#if UDP_BENCHMARK_TRACE 460 errval_t err = trace_control(TRACE_EVENT(TRACE_SUBSYS_BNET, 461 TRACE_EVENT_BNET_START, 0), 462 TRACE_EVENT(TRACE_SUBSYS_BNET, 463 TRACE_EVENT_BNET_STOP, 0), 0); 464 if(err_is_fail(err)) { 465 USER_PANIC_ERR(err, "trace_control failed"); 466 } 467 printf("U: Tracing enabled!!!!\n"); 468// trace_event(TRACE_SUBSYS_BNET, TRACE_EVENT_BNET_START, 0); 469#endif // UDP_BENCHMARK_TRACE 470 471 if(as_sender == 1) { 472 udp_sender(upcb, peer_ip, port); 473 } else { 474 udp_receiver(upcb, IP_ADDR_ANY, port); 475 } // end else: 476 477 478 printf("U: Init finished.\n"); 479 480 while (1) { 481 errval_t r = event_dispatch(ws); 482 if (err_is_fail(r)) { 483 DEBUG_ERR(r, "in event_dispatch"); 484 break; 485 } 486 } 487 488 udp_remove(upcb); 489} // end function: main 490 491