1321936Shselasky/* 2321936Shselasky * Copyright (c) 2011-2012 Intel Corporation. All rights reserved. 3321936Shselasky * Copyright (c) 2014-2015 Mellanox Technologies LTD. All rights reserved. 4321936Shselasky * 5321936Shselasky * This software is available to you under the OpenIB.org BSD license 6321936Shselasky * below: 7321936Shselasky * 8321936Shselasky * Redistribution and use in source and binary forms, with or 9321936Shselasky * without modification, are permitted provided that the following 10321936Shselasky * conditions are met: 11321936Shselasky * 12321936Shselasky * - Redistributions of source code must retain the above 13321936Shselasky * copyright notice, this list of conditions and the following 14321936Shselasky * disclaimer. 15321936Shselasky * 16321936Shselasky * - Redistributions in binary form must reproduce the above 17321936Shselasky * copyright notice, this list of conditions and the following 18321936Shselasky * disclaimer in the documentation and/or other materials 19321936Shselasky * provided with the distribution. 20321936Shselasky * 21321936Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22321936Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 23321936Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV 24321936Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 25321936Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 26321936Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 27321936Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28321936Shselasky * SOFTWARE. 29321936Shselasky */ 30321936Shselasky 31321936Shselasky#include <stdio.h> 32321936Shselasky#include <stdlib.h> 33321936Shselasky#include <string.h> 34321936Shselasky#include <strings.h> 35321936Shselasky#include <errno.h> 36321936Shselasky#include <getopt.h> 37321936Shselasky#include <sys/types.h> 38321936Shselasky#include <sys/socket.h> 39321936Shselasky#include <sys/time.h> 40321936Shselasky#include <sys/wait.h> 41321936Shselasky#include <netdb.h> 42321936Shselasky#include <fcntl.h> 43321936Shselasky#include <unistd.h> 44321936Shselasky#include <netinet/tcp.h> 45321936Shselasky 46321936Shselasky#include <rdma/rdma_cma.h> 47321936Shselasky#include <rdma/rsocket.h> 48321936Shselasky#include <util/compiler.h> 49321936Shselasky#include "common.h" 50321936Shselasky 51321936Shselaskystruct test_size_param { 52321936Shselasky int size; 53321936Shselasky int option; 54321936Shselasky}; 55321936Shselasky 56321936Shselaskystatic struct test_size_param test_size[] = { 57321936Shselasky { 1 << 6, 0 }, 58321936Shselasky { 1 << 7, 1 }, { (1 << 7) + (1 << 6), 1}, 59321936Shselasky { 1 << 8, 1 }, { (1 << 8) + (1 << 7), 1}, 60321936Shselasky { 1 << 9, 1 }, { (1 << 9) + (1 << 8), 1}, 61321936Shselasky { 1 << 10, 1 }, { (1 << 10) + (1 << 9), 1}, 62321936Shselasky { 1 << 11, 1 }, { (1 << 11) + (1 << 10), 1}, 63321936Shselasky { 1 << 12, 0 }, { (1 << 12) + (1 << 11), 1}, 64321936Shselasky { 1 << 13, 1 }, { (1 << 13) + (1 << 12), 1}, 65321936Shselasky { 1 << 14, 1 }, { (1 << 14) + (1 << 13), 1}, 66321936Shselasky { 1 << 15, 1 }, { (1 << 15) + (1 << 14), 1}, 67321936Shselasky { 1 << 16, 0 }, { (1 << 16) + (1 << 15), 1}, 68321936Shselasky { 1 << 17, 1 }, { (1 << 17) + (1 << 16), 1}, 69321936Shselasky { 1 << 18, 1 }, { (1 << 18) + (1 << 17), 1}, 70321936Shselasky { 1 << 19, 1 }, { (1 << 19) + (1 << 18), 1}, 71321936Shselasky { 1 << 20, 0 }, { (1 << 20) + (1 << 19), 1}, 72321936Shselasky { 1 << 21, 1 }, { (1 << 21) + (1 << 20), 1}, 73321936Shselasky { 1 << 22, 1 }, { (1 << 22) + (1 << 21), 1}, 74321936Shselasky}; 75321936Shselasky#define TEST_CNT (sizeof test_size / sizeof test_size[0]) 76321936Shselasky 77321936Shselaskystatic int rs, lrs; 78321936Shselaskystatic int use_async; 79321936Shselaskystatic int use_rgai; 80321936Shselaskystatic int verify; 81321936Shselaskystatic int flags = MSG_DONTWAIT; 82321936Shselaskystatic int poll_timeout = 0; 83321936Shselaskystatic int custom; 84321936Shselaskystatic int use_fork; 85321936Shselaskystatic pid_t fork_pid; 86321936Shselaskystatic enum rs_optimization optimization; 87321936Shselaskystatic int size_option; 88321936Shselaskystatic int iterations = 1; 89321936Shselaskystatic int transfer_size = 1000; 90321936Shselaskystatic int transfer_count = 1000; 91321936Shselaskystatic int buffer_size, inline_size = 64; 92321936Shselaskystatic char test_name[10] = "custom"; 93321936Shselaskystatic const char *port = "7471"; 94321936Shselaskystatic int keepalive; 95321936Shselaskystatic char *dst_addr; 96321936Shselaskystatic char *src_addr; 97321936Shselaskystatic struct timeval start, end; 98321936Shselaskystatic void *buf; 99321936Shselaskystatic struct rdma_addrinfo rai_hints; 100321936Shselaskystatic struct addrinfo ai_hints; 101321936Shselasky 102321936Shselaskystatic void show_perf(void) 103321936Shselasky{ 104321936Shselasky char str[32]; 105321936Shselasky float usec; 106321936Shselasky long long bytes; 107321936Shselasky 108321936Shselasky usec = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec); 109321936Shselasky bytes = (long long) iterations * transfer_count * transfer_size * 2; 110321936Shselasky 111321936Shselasky /* name size transfers iterations bytes seconds Gb/sec usec/xfer */ 112321936Shselasky printf("%-10s", test_name); 113321936Shselasky size_str(str, sizeof str, transfer_size); 114321936Shselasky printf("%-8s", str); 115321936Shselasky cnt_str(str, sizeof str, transfer_count); 116321936Shselasky printf("%-8s", str); 117321936Shselasky cnt_str(str, sizeof str, iterations); 118321936Shselasky printf("%-8s", str); 119321936Shselasky size_str(str, sizeof str, bytes); 120321936Shselasky printf("%-8s", str); 121321936Shselasky printf("%8.2fs%10.2f%11.2f\n", 122321936Shselasky usec / 1000000., (bytes * 8) / (1000. * usec), 123321936Shselasky (usec / iterations) / (transfer_count * 2)); 124321936Shselasky} 125321936Shselasky 126321936Shselaskystatic void init_latency_test(int size) 127321936Shselasky{ 128321936Shselasky char sstr[5]; 129321936Shselasky 130321936Shselasky size_str(sstr, sizeof sstr, size); 131321936Shselasky snprintf(test_name, sizeof test_name, "%s_lat", sstr); 132321936Shselasky transfer_count = 1; 133321936Shselasky transfer_size = size; 134321936Shselasky iterations = size_to_count(transfer_size); 135321936Shselasky} 136321936Shselasky 137321936Shselaskystatic void init_bandwidth_test(int size) 138321936Shselasky{ 139321936Shselasky char sstr[5]; 140321936Shselasky 141321936Shselasky size_str(sstr, sizeof sstr, size); 142321936Shselasky snprintf(test_name, sizeof test_name, "%s_bw", sstr); 143321936Shselasky iterations = 1; 144321936Shselasky transfer_size = size; 145321936Shselasky transfer_count = size_to_count(transfer_size); 146321936Shselasky} 147321936Shselasky 148321936Shselaskystatic int send_xfer(int size) 149321936Shselasky{ 150321936Shselasky struct pollfd fds; 151321936Shselasky int offset, ret; 152321936Shselasky 153321936Shselasky if (verify) 154321936Shselasky format_buf(buf, size); 155321936Shselasky 156321936Shselasky if (use_async) { 157321936Shselasky fds.fd = rs; 158321936Shselasky fds.events = POLLOUT; 159321936Shselasky } 160321936Shselasky 161321936Shselasky for (offset = 0; offset < size; ) { 162321936Shselasky if (use_async) { 163321936Shselasky ret = do_poll(&fds, poll_timeout); 164321936Shselasky if (ret) 165321936Shselasky return ret; 166321936Shselasky } 167321936Shselasky 168321936Shselasky ret = rs_send(rs, buf + offset, size - offset, flags); 169321936Shselasky if (ret > 0) { 170321936Shselasky offset += ret; 171321936Shselasky } else if (errno != EWOULDBLOCK && errno != EAGAIN) { 172321936Shselasky perror("rsend"); 173321936Shselasky return ret; 174321936Shselasky } 175321936Shselasky } 176321936Shselasky 177321936Shselasky return 0; 178321936Shselasky} 179321936Shselasky 180321936Shselaskystatic int recv_xfer(int size) 181321936Shselasky{ 182321936Shselasky struct pollfd fds; 183321936Shselasky int offset, ret; 184321936Shselasky 185321936Shselasky if (use_async) { 186321936Shselasky fds.fd = rs; 187321936Shselasky fds.events = POLLIN; 188321936Shselasky } 189321936Shselasky 190321936Shselasky for (offset = 0; offset < size; ) { 191321936Shselasky if (use_async) { 192321936Shselasky ret = do_poll(&fds, poll_timeout); 193321936Shselasky if (ret) 194321936Shselasky return ret; 195321936Shselasky } 196321936Shselasky 197321936Shselasky ret = rs_recv(rs, buf + offset, size - offset, flags); 198321936Shselasky if (ret > 0) { 199321936Shselasky offset += ret; 200321936Shselasky } else if (errno != EWOULDBLOCK && errno != EAGAIN) { 201321936Shselasky perror("rrecv"); 202321936Shselasky return ret; 203321936Shselasky } 204321936Shselasky } 205321936Shselasky 206321936Shselasky if (verify) { 207321936Shselasky ret = verify_buf(buf, size); 208321936Shselasky if (ret) 209321936Shselasky return ret; 210321936Shselasky } 211321936Shselasky 212321936Shselasky return 0; 213321936Shselasky} 214321936Shselasky 215321936Shselaskystatic int sync_test(void) 216321936Shselasky{ 217321936Shselasky int ret; 218321936Shselasky 219321936Shselasky ret = dst_addr ? send_xfer(16) : recv_xfer(16); 220321936Shselasky if (ret) 221321936Shselasky return ret; 222321936Shselasky 223321936Shselasky return dst_addr ? recv_xfer(16) : send_xfer(16); 224321936Shselasky} 225321936Shselasky 226321936Shselaskystatic int run_test(void) 227321936Shselasky{ 228321936Shselasky int ret, i, t; 229321936Shselasky 230321936Shselasky ret = sync_test(); 231321936Shselasky if (ret) 232321936Shselasky goto out; 233321936Shselasky 234321936Shselasky gettimeofday(&start, NULL); 235321936Shselasky for (i = 0; i < iterations; i++) { 236321936Shselasky for (t = 0; t < transfer_count; t++) { 237321936Shselasky ret = dst_addr ? send_xfer(transfer_size) : 238321936Shselasky recv_xfer(transfer_size); 239321936Shselasky if (ret) 240321936Shselasky goto out; 241321936Shselasky } 242321936Shselasky 243321936Shselasky for (t = 0; t < transfer_count; t++) { 244321936Shselasky ret = dst_addr ? recv_xfer(transfer_size) : 245321936Shselasky send_xfer(transfer_size); 246321936Shselasky if (ret) 247321936Shselasky goto out; 248321936Shselasky } 249321936Shselasky } 250321936Shselasky gettimeofday(&end, NULL); 251321936Shselasky show_perf(); 252321936Shselasky ret = 0; 253321936Shselasky 254321936Shselaskyout: 255321936Shselasky return ret; 256321936Shselasky} 257321936Shselasky 258321936Shselaskystatic void set_keepalive(int fd) 259321936Shselasky{ 260321936Shselasky int optval; 261321936Shselasky socklen_t optlen = sizeof(optlen); 262321936Shselasky 263321936Shselasky optval = 1; 264321936Shselasky if (rs_setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen)) { 265321936Shselasky perror("rsetsockopt SO_KEEPALIVE"); 266321936Shselasky return; 267321936Shselasky } 268321936Shselasky 269321936Shselasky optval = keepalive; 270321936Shselasky if (rs_setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &optval, optlen)) 271321936Shselasky perror("rsetsockopt TCP_KEEPIDLE"); 272321936Shselasky 273321936Shselasky if (!(rs_getsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen))) 274321936Shselasky printf("Keepalive: %s\n", (optval ? "ON" : "OFF")); 275321936Shselasky 276321936Shselasky if (!(rs_getsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &optval, &optlen))) 277321936Shselasky printf(" time: %i\n", optval); 278321936Shselasky} 279321936Shselasky 280321936Shselaskystatic void set_options(int fd) 281321936Shselasky{ 282321936Shselasky int val; 283321936Shselasky 284321936Shselasky if (buffer_size) { 285321936Shselasky rs_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (void *) &buffer_size, 286321936Shselasky sizeof buffer_size); 287321936Shselasky rs_setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (void *) &buffer_size, 288321936Shselasky sizeof buffer_size); 289321936Shselasky } else { 290321936Shselasky val = 1 << 19; 291321936Shselasky rs_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (void *) &val, sizeof val); 292321936Shselasky rs_setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (void *) &val, sizeof val); 293321936Shselasky } 294321936Shselasky 295321936Shselasky val = 1; 296321936Shselasky rs_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (void *) &val, sizeof(val)); 297321936Shselasky 298321936Shselasky if (flags & MSG_DONTWAIT) 299321936Shselasky rs_fcntl(fd, F_SETFL, O_NONBLOCK); 300321936Shselasky 301321936Shselasky if (use_rs) { 302321936Shselasky /* Inline size based on experimental data */ 303321936Shselasky if (optimization == opt_latency) { 304321936Shselasky rs_setsockopt(fd, SOL_RDMA, RDMA_INLINE, &inline_size, 305321936Shselasky sizeof inline_size); 306321936Shselasky } else if (optimization == opt_bandwidth) { 307321936Shselasky val = 0; 308321936Shselasky rs_setsockopt(fd, SOL_RDMA, RDMA_INLINE, &val, sizeof val); 309321936Shselasky } 310321936Shselasky } 311321936Shselasky 312321936Shselasky if (keepalive) 313321936Shselasky set_keepalive(fd); 314321936Shselasky} 315321936Shselasky 316321936Shselaskystatic int server_listen(void) 317321936Shselasky{ 318321936Shselasky struct rdma_addrinfo *rai = NULL; 319321936Shselasky struct addrinfo *ai; 320321936Shselasky int val, ret; 321321936Shselasky 322321936Shselasky if (use_rgai) { 323321936Shselasky rai_hints.ai_flags |= RAI_PASSIVE; 324321936Shselasky ret = rdma_getaddrinfo(src_addr, port, &rai_hints, &rai); 325321936Shselasky } else { 326321936Shselasky ai_hints.ai_flags |= AI_PASSIVE; 327321936Shselasky ret = getaddrinfo(src_addr, port, &ai_hints, &ai); 328321936Shselasky } 329321936Shselasky if (ret) { 330321936Shselasky printf("getaddrinfo: %s\n", gai_strerror(ret)); 331321936Shselasky return ret; 332321936Shselasky } 333321936Shselasky 334321936Shselasky lrs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) : 335321936Shselasky rs_socket(ai->ai_family, SOCK_STREAM, 0); 336321936Shselasky if (lrs < 0) { 337321936Shselasky perror("rsocket"); 338321936Shselasky ret = lrs; 339321936Shselasky goto free; 340321936Shselasky } 341321936Shselasky 342321936Shselasky val = 1; 343321936Shselasky ret = rs_setsockopt(lrs, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val); 344321936Shselasky if (ret) { 345321936Shselasky perror("rsetsockopt SO_REUSEADDR"); 346321936Shselasky goto close; 347321936Shselasky } 348321936Shselasky 349321936Shselasky ret = rai ? rs_bind(lrs, rai->ai_src_addr, rai->ai_src_len) : 350321936Shselasky rs_bind(lrs, ai->ai_addr, ai->ai_addrlen); 351321936Shselasky if (ret) { 352321936Shselasky perror("rbind"); 353321936Shselasky goto close; 354321936Shselasky } 355321936Shselasky 356321936Shselasky ret = rs_listen(lrs, 1); 357321936Shselasky if (ret) 358321936Shselasky perror("rlisten"); 359321936Shselasky 360321936Shselaskyclose: 361321936Shselasky if (ret) 362321936Shselasky rs_close(lrs); 363321936Shselaskyfree: 364321936Shselasky if (rai) 365321936Shselasky rdma_freeaddrinfo(rai); 366321936Shselasky else 367321936Shselasky freeaddrinfo(ai); 368321936Shselasky return ret; 369321936Shselasky} 370321936Shselasky 371321936Shselaskystatic int server_connect(void) 372321936Shselasky{ 373321936Shselasky struct pollfd fds; 374321936Shselasky int ret = 0; 375321936Shselasky 376321936Shselasky set_options(lrs); 377321936Shselasky do { 378321936Shselasky if (use_async) { 379321936Shselasky fds.fd = lrs; 380321936Shselasky fds.events = POLLIN; 381321936Shselasky 382321936Shselasky ret = do_poll(&fds, poll_timeout); 383321936Shselasky if (ret) { 384321936Shselasky perror("rpoll"); 385321936Shselasky return ret; 386321936Shselasky } 387321936Shselasky } 388321936Shselasky 389321936Shselasky rs = rs_accept(lrs, NULL, NULL); 390321936Shselasky } while (rs < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)); 391321936Shselasky if (rs < 0) { 392321936Shselasky perror("raccept"); 393321936Shselasky return rs; 394321936Shselasky } 395321936Shselasky 396321936Shselasky if (use_fork) 397321936Shselasky fork_pid = fork(); 398321936Shselasky if (!fork_pid) 399321936Shselasky set_options(rs); 400321936Shselasky return ret; 401321936Shselasky} 402321936Shselasky 403321936Shselaskystatic int client_connect(void) 404321936Shselasky{ 405321936Shselasky struct rdma_addrinfo *rai = NULL, *rai_src = NULL; 406321936Shselasky struct addrinfo *ai, *ai_src; 407321936Shselasky struct pollfd fds; 408321936Shselasky int ret, err; 409321936Shselasky socklen_t len; 410321936Shselasky 411321936Shselasky ret = use_rgai ? rdma_getaddrinfo(dst_addr, port, &rai_hints, &rai) : 412321936Shselasky getaddrinfo(dst_addr, port, &ai_hints, &ai); 413321936Shselasky 414321936Shselasky if (ret) { 415321936Shselasky printf("getaddrinfo: %s\n", gai_strerror(ret)); 416321936Shselasky return ret; 417321936Shselasky } 418321936Shselasky 419321936Shselasky if (src_addr) { 420321936Shselasky if (use_rgai) { 421321936Shselasky rai_hints.ai_flags |= RAI_PASSIVE; 422321936Shselasky ret = rdma_getaddrinfo(src_addr, port, &rai_hints, &rai_src); 423321936Shselasky } else { 424321936Shselasky ai_hints.ai_flags |= AI_PASSIVE; 425321936Shselasky ret = getaddrinfo(src_addr, port, &ai_hints, &ai_src); 426321936Shselasky } 427321936Shselasky if (ret) { 428321936Shselasky printf("getaddrinfo src_addr: %s\n", gai_strerror(ret)); 429321936Shselasky return ret; 430321936Shselasky } 431321936Shselasky } 432321936Shselasky 433321936Shselasky rs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) : 434321936Shselasky rs_socket(ai->ai_family, SOCK_STREAM, 0); 435321936Shselasky if (rs < 0) { 436321936Shselasky perror("rsocket"); 437321936Shselasky ret = rs; 438321936Shselasky goto free; 439321936Shselasky } 440321936Shselasky 441321936Shselasky set_options(rs); 442321936Shselasky 443321936Shselasky if (src_addr) { 444321936Shselasky ret = rai ? rs_bind(rs, rai_src->ai_src_addr, rai_src->ai_src_len) : 445321936Shselasky rs_bind(rs, ai_src->ai_addr, ai_src->ai_addrlen); 446321936Shselasky if (ret) { 447321936Shselasky perror("rbind"); 448321936Shselasky goto close; 449321936Shselasky } 450321936Shselasky } 451321936Shselasky 452321936Shselasky if (rai && rai->ai_route) { 453321936Shselasky ret = rs_setsockopt(rs, SOL_RDMA, RDMA_ROUTE, rai->ai_route, 454321936Shselasky rai->ai_route_len); 455321936Shselasky if (ret) { 456321936Shselasky perror("rsetsockopt RDMA_ROUTE"); 457321936Shselasky goto close; 458321936Shselasky } 459321936Shselasky } 460321936Shselasky 461321936Shselasky ret = rai ? rs_connect(rs, rai->ai_dst_addr, rai->ai_dst_len) : 462321936Shselasky rs_connect(rs, ai->ai_addr, ai->ai_addrlen); 463321936Shselasky if (ret && (errno != EINPROGRESS)) { 464321936Shselasky perror("rconnect"); 465321936Shselasky goto close; 466321936Shselasky } 467321936Shselasky 468321936Shselasky if (ret && (errno == EINPROGRESS)) { 469321936Shselasky fds.fd = rs; 470321936Shselasky fds.events = POLLOUT; 471321936Shselasky ret = do_poll(&fds, poll_timeout); 472321936Shselasky if (ret) { 473321936Shselasky perror("rpoll"); 474321936Shselasky goto close; 475321936Shselasky } 476321936Shselasky 477321936Shselasky len = sizeof err; 478321936Shselasky ret = rs_getsockopt(rs, SOL_SOCKET, SO_ERROR, &err, &len); 479321936Shselasky if (ret) 480321936Shselasky goto close; 481321936Shselasky if (err) { 482321936Shselasky ret = -1; 483321936Shselasky errno = err; 484321936Shselasky perror("async rconnect"); 485321936Shselasky } 486321936Shselasky } 487321936Shselasky 488321936Shselaskyclose: 489321936Shselasky if (ret) 490321936Shselasky rs_close(rs); 491321936Shselaskyfree: 492321936Shselasky if (rai) 493321936Shselasky rdma_freeaddrinfo(rai); 494321936Shselasky else 495321936Shselasky freeaddrinfo(ai); 496321936Shselasky return ret; 497321936Shselasky} 498321936Shselasky 499321936Shselaskystatic int run(void) 500321936Shselasky{ 501321936Shselasky int i, ret = 0; 502321936Shselasky 503321936Shselasky buf = malloc(!custom ? test_size[TEST_CNT - 1].size : transfer_size); 504321936Shselasky if (!buf) { 505321936Shselasky perror("malloc"); 506321936Shselasky return -1; 507321936Shselasky } 508321936Shselasky 509321936Shselasky if (!dst_addr) { 510321936Shselasky ret = server_listen(); 511321936Shselasky if (ret) 512321936Shselasky goto free; 513321936Shselasky } 514321936Shselasky 515321936Shselasky printf("%-10s%-8s%-8s%-8s%-8s%8s %10s%13s\n", 516321936Shselasky "name", "bytes", "xfers", "iters", "total", "time", "Gb/sec", "usec/xfer"); 517321936Shselasky if (!custom) { 518321936Shselasky optimization = opt_latency; 519321936Shselasky ret = dst_addr ? client_connect() : server_connect(); 520321936Shselasky if (ret) 521321936Shselasky goto free; 522321936Shselasky 523321936Shselasky for (i = 0; i < TEST_CNT && !fork_pid; i++) { 524321936Shselasky if (test_size[i].option > size_option) 525321936Shselasky continue; 526321936Shselasky init_latency_test(test_size[i].size); 527321936Shselasky run_test(); 528321936Shselasky } 529321936Shselasky if (fork_pid) 530321936Shselasky waitpid(fork_pid, NULL, 0); 531321936Shselasky else 532321936Shselasky rs_shutdown(rs, SHUT_RDWR); 533321936Shselasky rs_close(rs); 534321936Shselasky 535321936Shselasky if (!dst_addr && use_fork && !fork_pid) 536321936Shselasky goto free; 537321936Shselasky 538321936Shselasky optimization = opt_bandwidth; 539321936Shselasky ret = dst_addr ? client_connect() : server_connect(); 540321936Shselasky if (ret) 541321936Shselasky goto free; 542321936Shselasky for (i = 0; i < TEST_CNT && !fork_pid; i++) { 543321936Shselasky if (test_size[i].option > size_option) 544321936Shselasky continue; 545321936Shselasky init_bandwidth_test(test_size[i].size); 546321936Shselasky run_test(); 547321936Shselasky } 548321936Shselasky } else { 549321936Shselasky ret = dst_addr ? client_connect() : server_connect(); 550321936Shselasky if (ret) 551321936Shselasky goto free; 552321936Shselasky 553321936Shselasky if (!fork_pid) 554321936Shselasky ret = run_test(); 555321936Shselasky } 556321936Shselasky 557321936Shselasky if (fork_pid) 558321936Shselasky waitpid(fork_pid, NULL, 0); 559321936Shselasky else 560321936Shselasky rs_shutdown(rs, SHUT_RDWR); 561321936Shselasky rs_close(rs); 562321936Shselaskyfree: 563321936Shselasky free(buf); 564321936Shselasky return ret; 565321936Shselasky} 566321936Shselasky 567321936Shselaskystatic int set_test_opt(const char *arg) 568321936Shselasky{ 569321936Shselasky if (strlen(arg) == 1) { 570321936Shselasky switch (arg[0]) { 571321936Shselasky case 's': 572321936Shselasky use_rs = 0; 573321936Shselasky break; 574321936Shselasky case 'a': 575321936Shselasky use_async = 1; 576321936Shselasky break; 577321936Shselasky case 'b': 578321936Shselasky flags = (flags & ~MSG_DONTWAIT) | MSG_WAITALL; 579321936Shselasky break; 580321936Shselasky case 'f': 581321936Shselasky use_fork = 1; 582321936Shselasky use_rs = 0; 583321936Shselasky break; 584321936Shselasky case 'n': 585321936Shselasky flags |= MSG_DONTWAIT; 586321936Shselasky break; 587321936Shselasky case 'r': 588321936Shselasky use_rgai = 1; 589321936Shselasky break; 590321936Shselasky case 'v': 591321936Shselasky verify = 1; 592321936Shselasky break; 593321936Shselasky default: 594321936Shselasky return -1; 595321936Shselasky } 596321936Shselasky } else { 597321936Shselasky if (!strncasecmp("socket", arg, 6)) { 598321936Shselasky use_rs = 0; 599321936Shselasky } else if (!strncasecmp("async", arg, 5)) { 600321936Shselasky use_async = 1; 601321936Shselasky } else if (!strncasecmp("block", arg, 5)) { 602321936Shselasky flags = (flags & ~MSG_DONTWAIT) | MSG_WAITALL; 603321936Shselasky } else if (!strncasecmp("nonblock", arg, 8)) { 604321936Shselasky flags |= MSG_DONTWAIT; 605321936Shselasky } else if (!strncasecmp("resolve", arg, 7)) { 606321936Shselasky use_rgai = 1; 607321936Shselasky } else if (!strncasecmp("verify", arg, 6)) { 608321936Shselasky verify = 1; 609321936Shselasky } else if (!strncasecmp("fork", arg, 4)) { 610321936Shselasky use_fork = 1; 611321936Shselasky use_rs = 0; 612321936Shselasky } else { 613321936Shselasky return -1; 614321936Shselasky } 615321936Shselasky } 616321936Shselasky return 0; 617321936Shselasky} 618321936Shselasky 619321936Shselaskyint main(int argc, char **argv) 620321936Shselasky{ 621321936Shselasky int op, ret; 622321936Shselasky 623321936Shselasky ai_hints.ai_socktype = SOCK_STREAM; 624321936Shselasky rai_hints.ai_port_space = RDMA_PS_TCP; 625321936Shselasky while ((op = getopt(argc, argv, "s:b:f:B:i:I:C:S:p:k:T:")) != -1) { 626321936Shselasky switch (op) { 627321936Shselasky case 's': 628321936Shselasky dst_addr = optarg; 629321936Shselasky break; 630321936Shselasky case 'b': 631321936Shselasky src_addr = optarg; 632321936Shselasky break; 633321936Shselasky case 'f': 634321936Shselasky if (!strncasecmp("ip", optarg, 2)) { 635321936Shselasky ai_hints.ai_flags = AI_NUMERICHOST; 636321936Shselasky } else if (!strncasecmp("gid", optarg, 3)) { 637321936Shselasky rai_hints.ai_flags = RAI_NUMERICHOST | RAI_FAMILY; 638321936Shselasky rai_hints.ai_family = AF_IB; 639321936Shselasky use_rgai = 1; 640321936Shselasky } else { 641321936Shselasky fprintf(stderr, "Warning: unknown address format\n"); 642321936Shselasky } 643321936Shselasky break; 644321936Shselasky case 'B': 645321936Shselasky buffer_size = atoi(optarg); 646321936Shselasky break; 647321936Shselasky case 'i': 648321936Shselasky inline_size = atoi(optarg); 649321936Shselasky break; 650321936Shselasky case 'I': 651321936Shselasky custom = 1; 652321936Shselasky iterations = atoi(optarg); 653321936Shselasky break; 654321936Shselasky case 'C': 655321936Shselasky custom = 1; 656321936Shselasky transfer_count = atoi(optarg); 657321936Shselasky break; 658321936Shselasky case 'S': 659321936Shselasky if (!strncasecmp("all", optarg, 3)) { 660321936Shselasky size_option = 1; 661321936Shselasky } else { 662321936Shselasky custom = 1; 663321936Shselasky transfer_size = atoi(optarg); 664321936Shselasky } 665321936Shselasky break; 666321936Shselasky case 'p': 667321936Shselasky port = optarg; 668321936Shselasky break; 669321936Shselasky case 'k': 670321936Shselasky keepalive = atoi(optarg); 671321936Shselasky break; 672321936Shselasky case 'T': 673321936Shselasky if (!set_test_opt(optarg)) 674321936Shselasky break; 675321936Shselasky /* invalid option - fall through */ 676321936Shselasky SWITCH_FALLTHROUGH; 677321936Shselasky default: 678321936Shselasky printf("usage: %s\n", argv[0]); 679321936Shselasky printf("\t[-s server_address]\n"); 680321936Shselasky printf("\t[-b bind_address]\n"); 681321936Shselasky printf("\t[-f address_format]\n"); 682321936Shselasky printf("\t name, ip, ipv6, or gid\n"); 683321936Shselasky printf("\t[-B buffer_size]\n"); 684321936Shselasky printf("\t[-i inline_size]\n"); 685321936Shselasky printf("\t[-I iterations]\n"); 686321936Shselasky printf("\t[-C transfer_count]\n"); 687321936Shselasky printf("\t[-S transfer_size or all]\n"); 688321936Shselasky printf("\t[-p port_number]\n"); 689321936Shselasky printf("\t[-k keepalive_time]\n"); 690321936Shselasky printf("\t[-T test_option]\n"); 691321936Shselasky printf("\t s|sockets - use standard tcp/ip sockets\n"); 692321936Shselasky printf("\t a|async - asynchronous operation (use poll)\n"); 693321936Shselasky printf("\t b|blocking - use blocking calls\n"); 694321936Shselasky printf("\t f|fork - fork server processing\n"); 695321936Shselasky printf("\t n|nonblocking - use nonblocking calls\n"); 696321936Shselasky printf("\t r|resolve - use rdma cm to resolve address\n"); 697321936Shselasky printf("\t v|verify - verify data\n"); 698321936Shselasky exit(1); 699321936Shselasky } 700321936Shselasky } 701321936Shselasky 702321936Shselasky if (!(flags & MSG_DONTWAIT)) 703321936Shselasky poll_timeout = -1; 704321936Shselasky 705321936Shselasky ret = run(); 706321936Shselasky return ret; 707321936Shselasky} 708