1321936Shselasky/* 2321936Shselasky * Copyright (c) 2013 Intel Corporation. All rights reserved. 3321936Shselasky * 4321936Shselasky * This software is available to you under the OpenIB.org BSD license 5321936Shselasky * below: 6321936Shselasky * 7321936Shselasky * Redistribution and use in source and binary forms, with or 8321936Shselasky * without modification, are permitted provided that the following 9321936Shselasky * conditions are met: 10321936Shselasky * 11321936Shselasky * - Redistributions of source code must retain the above 12321936Shselasky * copyright notice, this list of conditions and the following 13321936Shselasky * disclaimer. 14321936Shselasky * 15321936Shselasky * - Redistributions in binary form must reproduce the above 16321936Shselasky * copyright notice, this list of conditions and the following 17321936Shselasky * disclaimer in the documentation and/or other materials 18321936Shselasky * provided with the distribution. 19321936Shselasky * 20321936Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21321936Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22321936Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV 23321936Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 24321936Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 25321936Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 26321936Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 27321936Shselasky * SOFTWARE. 28321936Shselasky */ 29321936Shselasky 30321936Shselasky#include <stdio.h> 31321936Shselasky#include <stdlib.h> 32321936Shselasky#include <string.h> 33321936Shselasky#include <strings.h> 34321936Shselasky#include <errno.h> 35321936Shselasky#include <getopt.h> 36321936Shselasky#include <sys/types.h> 37321936Shselasky#include <sys/socket.h> 38321936Shselasky#include <sys/time.h> 39321936Shselasky#include <sys/wait.h> 40321936Shselasky#include <netdb.h> 41321936Shselasky#include <fcntl.h> 42321936Shselasky#include <unistd.h> 43321936Shselasky#include <netinet/tcp.h> 44321936Shselasky 45321936Shselasky#include <rdma/rdma_cma.h> 46321936Shselasky#include "common.h" 47321936Shselasky 48321936Shselaskystatic struct rdma_addrinfo hints, *rai; 49321936Shselaskystatic struct rdma_event_channel *channel; 50321936Shselaskystatic const char *port = "7471"; 51321936Shselaskystatic char *dst_addr; 52321936Shselaskystatic char *src_addr; 53321936Shselaskystatic int timeout = 2000; 54321936Shselaskystatic int retries = 2; 55321936Shselasky 56321936Shselaskyenum step { 57321936Shselasky STEP_CREATE_ID, 58321936Shselasky STEP_BIND, 59321936Shselasky STEP_RESOLVE_ADDR, 60321936Shselasky STEP_RESOLVE_ROUTE, 61321936Shselasky STEP_CREATE_QP, 62321936Shselasky STEP_CONNECT, 63321936Shselasky STEP_DISCONNECT, 64321936Shselasky STEP_DESTROY, 65321936Shselasky STEP_CNT 66321936Shselasky}; 67321936Shselasky 68321936Shselaskystatic const char *step_str[] = { 69321936Shselasky "create id", 70321936Shselasky "bind addr", 71321936Shselasky "resolve addr", 72321936Shselasky "resolve route", 73321936Shselasky "create qp", 74321936Shselasky "connect", 75321936Shselasky "disconnect", 76321936Shselasky "destroy" 77321936Shselasky}; 78321936Shselasky 79321936Shselaskystruct node { 80321936Shselasky struct rdma_cm_id *id; 81321936Shselasky struct timeval times[STEP_CNT][2]; 82321936Shselasky int error; 83321936Shselasky int retries; 84321936Shselasky}; 85321936Shselasky 86321936Shselaskystruct list_head { 87321936Shselasky struct list_head *prev; 88321936Shselasky struct list_head *next; 89321936Shselasky struct rdma_cm_id *id; 90321936Shselasky}; 91321936Shselasky 92321936Shselaskystruct work_list { 93321936Shselasky pthread_mutex_t lock; 94321936Shselasky pthread_cond_t cond; 95321936Shselasky struct list_head list; 96321936Shselasky}; 97321936Shselasky 98321936Shselasky#define INIT_LIST(x) ((x)->prev = (x)->next = (x)) 99321936Shselasky 100321936Shselaskystatic struct work_list req_work; 101321936Shselaskystatic struct work_list disc_work; 102321936Shselaskystatic struct node *nodes; 103321936Shselaskystatic struct timeval times[STEP_CNT][2]; 104321936Shselaskystatic int connections = 100; 105321936Shselaskystatic volatile int started[STEP_CNT]; 106321936Shselaskystatic volatile int completed[STEP_CNT]; 107321936Shselaskystatic struct ibv_qp_init_attr init_qp_attr; 108321936Shselaskystatic struct rdma_conn_param conn_param; 109321936Shselasky 110321936Shselasky#define start_perf(n, s) gettimeofday(&((n)->times[s][0]), NULL) 111321936Shselasky#define end_perf(n, s) gettimeofday(&((n)->times[s][1]), NULL) 112321936Shselasky#define start_time(s) gettimeofday(×[s][0], NULL) 113321936Shselasky#define end_time(s) gettimeofday(×[s][1], NULL) 114321936Shselasky 115321936Shselaskystatic inline void __list_delete(struct list_head *list) 116321936Shselasky{ 117321936Shselasky struct list_head *prev, *next; 118321936Shselasky prev = list->prev; 119321936Shselasky next = list->next; 120321936Shselasky prev->next = next; 121321936Shselasky next->prev = prev; 122321936Shselasky INIT_LIST(list); 123321936Shselasky} 124321936Shselasky 125321936Shselaskystatic inline int __list_empty(struct work_list *list) 126321936Shselasky{ 127321936Shselasky return list->list.next == &list->list; 128321936Shselasky} 129321936Shselasky 130321936Shselaskystatic inline struct list_head *__list_remove_head(struct work_list *work_list) 131321936Shselasky{ 132321936Shselasky struct list_head *list_item; 133321936Shselasky 134321936Shselasky list_item = work_list->list.next; 135321936Shselasky __list_delete(list_item); 136321936Shselasky return list_item; 137321936Shselasky} 138321936Shselasky 139321936Shselaskystatic inline void list_add_tail(struct work_list *work_list, struct list_head *req) 140321936Shselasky{ 141321936Shselasky int empty; 142321936Shselasky pthread_mutex_lock(&work_list->lock); 143321936Shselasky empty = __list_empty(work_list); 144321936Shselasky req->prev = work_list->list.prev; 145321936Shselasky req->next = &work_list->list; 146321936Shselasky req->prev->next = work_list->list.prev = req; 147321936Shselasky pthread_mutex_unlock(&work_list->lock); 148321936Shselasky if (empty) 149321936Shselasky pthread_cond_signal(&work_list->cond); 150321936Shselasky} 151321936Shselasky 152321936Shselaskystatic int zero_time(struct timeval *t) 153321936Shselasky{ 154321936Shselasky return !(t->tv_sec || t->tv_usec); 155321936Shselasky} 156321936Shselasky 157321936Shselaskystatic float diff_us(struct timeval *end, struct timeval *start) 158321936Shselasky{ 159321936Shselasky return (end->tv_sec - start->tv_sec) * 1000000. + (end->tv_usec - start->tv_usec); 160321936Shselasky} 161321936Shselasky 162321936Shselaskystatic void show_perf(void) 163321936Shselasky{ 164321936Shselasky int c, i; 165321936Shselasky float us, max[STEP_CNT], min[STEP_CNT]; 166321936Shselasky 167321936Shselasky for (i = 0; i < STEP_CNT; i++) { 168321936Shselasky max[i] = 0; 169321936Shselasky min[i] = 999999999.; 170321936Shselasky for (c = 0; c < connections; c++) { 171321936Shselasky if (!zero_time(&nodes[c].times[i][0]) && 172321936Shselasky !zero_time(&nodes[c].times[i][1])) { 173321936Shselasky us = diff_us(&nodes[c].times[i][1], &nodes[c].times[i][0]); 174321936Shselasky if (us > max[i]) 175321936Shselasky max[i] = us; 176321936Shselasky if (us < min[i]) 177321936Shselasky min[i] = us; 178321936Shselasky } 179321936Shselasky } 180321936Shselasky } 181321936Shselasky 182321936Shselasky printf("step total ms max ms min us us / conn\n"); 183321936Shselasky for (i = 0; i < STEP_CNT; i++) { 184321936Shselasky if (i == STEP_BIND && !src_addr) 185321936Shselasky continue; 186321936Shselasky 187321936Shselasky us = diff_us(×[i][1], ×[i][0]); 188321936Shselasky printf("%-13s: %11.2f%11.2f%11.2f%11.2f\n", step_str[i], us / 1000., 189321936Shselasky max[i] / 1000., min[i], us / connections); 190321936Shselasky } 191321936Shselasky} 192321936Shselasky 193321936Shselaskystatic void addr_handler(struct node *n) 194321936Shselasky{ 195321936Shselasky end_perf(n, STEP_RESOLVE_ADDR); 196321936Shselasky completed[STEP_RESOLVE_ADDR]++; 197321936Shselasky} 198321936Shselasky 199321936Shselaskystatic void route_handler(struct node *n) 200321936Shselasky{ 201321936Shselasky end_perf(n, STEP_RESOLVE_ROUTE); 202321936Shselasky completed[STEP_RESOLVE_ROUTE]++; 203321936Shselasky} 204321936Shselasky 205321936Shselaskystatic void conn_handler(struct node *n) 206321936Shselasky{ 207321936Shselasky end_perf(n, STEP_CONNECT); 208321936Shselasky completed[STEP_CONNECT]++; 209321936Shselasky} 210321936Shselasky 211321936Shselaskystatic void disc_handler(struct node *n) 212321936Shselasky{ 213321936Shselasky end_perf(n, STEP_DISCONNECT); 214321936Shselasky completed[STEP_DISCONNECT]++; 215321936Shselasky} 216321936Shselasky 217321936Shselaskystatic void __req_handler(struct rdma_cm_id *id) 218321936Shselasky{ 219321936Shselasky int ret; 220321936Shselasky 221321936Shselasky ret = rdma_create_qp(id, NULL, &init_qp_attr); 222321936Shselasky if (ret) { 223321936Shselasky perror("failure creating qp"); 224321936Shselasky goto err; 225321936Shselasky } 226321936Shselasky 227321936Shselasky ret = rdma_accept(id, NULL); 228321936Shselasky if (ret) { 229321936Shselasky perror("failure accepting"); 230321936Shselasky goto err; 231321936Shselasky } 232321936Shselasky return; 233321936Shselasky 234321936Shselaskyerr: 235321936Shselasky printf("failing connection request\n"); 236321936Shselasky rdma_reject(id, NULL, 0); 237321936Shselasky rdma_destroy_id(id); 238321936Shselasky return; 239321936Shselasky} 240321936Shselasky 241321936Shselaskystatic void *req_handler_thread(void *arg) 242321936Shselasky{ 243321936Shselasky struct list_head *work; 244321936Shselasky do { 245321936Shselasky pthread_mutex_lock(&req_work.lock); 246321936Shselasky if (__list_empty(&req_work)) 247321936Shselasky pthread_cond_wait(&req_work.cond, &req_work.lock); 248321936Shselasky work = __list_remove_head(&req_work); 249321936Shselasky pthread_mutex_unlock(&req_work.lock); 250321936Shselasky __req_handler(work->id); 251321936Shselasky free(work); 252321936Shselasky } while (1); 253321936Shselasky return NULL; 254321936Shselasky} 255321936Shselasky 256321936Shselaskystatic void *disc_handler_thread(void *arg) 257321936Shselasky{ 258321936Shselasky struct list_head *work; 259321936Shselasky do { 260321936Shselasky pthread_mutex_lock(&disc_work.lock); 261321936Shselasky if (__list_empty(&disc_work)) 262321936Shselasky pthread_cond_wait(&disc_work.cond, &disc_work.lock); 263321936Shselasky work = __list_remove_head(&disc_work); 264321936Shselasky pthread_mutex_unlock(&disc_work.lock); 265321936Shselasky rdma_disconnect(work->id); 266321936Shselasky rdma_destroy_id(work->id); 267321936Shselasky free(work); 268321936Shselasky } while (1); 269321936Shselasky return NULL; 270321936Shselasky} 271321936Shselasky 272321936Shselaskystatic void cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) 273321936Shselasky{ 274321936Shselasky struct node *n = id->context; 275321936Shselasky struct list_head *request; 276321936Shselasky 277321936Shselasky switch (event->event) { 278321936Shselasky case RDMA_CM_EVENT_ADDR_RESOLVED: 279321936Shselasky addr_handler(n); 280321936Shselasky break; 281321936Shselasky case RDMA_CM_EVENT_ROUTE_RESOLVED: 282321936Shselasky route_handler(n); 283321936Shselasky break; 284321936Shselasky case RDMA_CM_EVENT_CONNECT_REQUEST: 285321936Shselasky request = malloc(sizeof *request); 286321936Shselasky if (!request) { 287321936Shselasky perror("out of memory accepting connect request"); 288321936Shselasky rdma_reject(id, NULL, 0); 289321936Shselasky rdma_destroy_id(id); 290321936Shselasky } else { 291321936Shselasky INIT_LIST(request); 292321936Shselasky request->id = id; 293321936Shselasky list_add_tail(&req_work, request); 294321936Shselasky } 295321936Shselasky break; 296321936Shselasky case RDMA_CM_EVENT_ESTABLISHED: 297321936Shselasky if (n) 298321936Shselasky conn_handler(n); 299321936Shselasky break; 300321936Shselasky case RDMA_CM_EVENT_ADDR_ERROR: 301321936Shselasky if (n->retries--) { 302321936Shselasky if (!rdma_resolve_addr(n->id, rai->ai_src_addr, 303321936Shselasky rai->ai_dst_addr, timeout)) 304321936Shselasky break; 305321936Shselasky } 306321936Shselasky printf("RDMA_CM_EVENT_ADDR_ERROR, error: %d\n", event->status); 307321936Shselasky addr_handler(n); 308321936Shselasky n->error = 1; 309321936Shselasky break; 310321936Shselasky case RDMA_CM_EVENT_ROUTE_ERROR: 311321936Shselasky if (n->retries--) { 312321936Shselasky if (!rdma_resolve_route(n->id, timeout)) 313321936Shselasky break; 314321936Shselasky } 315321936Shselasky printf("RDMA_CM_EVENT_ROUTE_ERROR, error: %d\n", event->status); 316321936Shselasky route_handler(n); 317321936Shselasky n->error = 1; 318321936Shselasky break; 319321936Shselasky case RDMA_CM_EVENT_CONNECT_ERROR: 320321936Shselasky case RDMA_CM_EVENT_UNREACHABLE: 321321936Shselasky case RDMA_CM_EVENT_REJECTED: 322321936Shselasky printf("event: %s, error: %d\n", 323321936Shselasky rdma_event_str(event->event), event->status); 324321936Shselasky conn_handler(n); 325321936Shselasky n->error = 1; 326321936Shselasky break; 327321936Shselasky case RDMA_CM_EVENT_DISCONNECTED: 328321936Shselasky if (!n) { 329321936Shselasky request = malloc(sizeof *request); 330321936Shselasky if (!request) { 331321936Shselasky perror("out of memory queueing disconnect request, handling synchronously"); 332321936Shselasky rdma_disconnect(id); 333321936Shselasky rdma_destroy_id(id); 334321936Shselasky } else { 335321936Shselasky INIT_LIST(request); 336321936Shselasky request->id = id; 337321936Shselasky list_add_tail(&disc_work, request); 338321936Shselasky } 339321936Shselasky } else 340321936Shselasky disc_handler(n); 341321936Shselasky break; 342321936Shselasky case RDMA_CM_EVENT_DEVICE_REMOVAL: 343321936Shselasky /* Cleanup will occur after test completes. */ 344321936Shselasky break; 345321936Shselasky default: 346321936Shselasky break; 347321936Shselasky } 348321936Shselasky rdma_ack_cm_event(event); 349321936Shselasky} 350321936Shselasky 351321936Shselaskystatic int alloc_nodes(void) 352321936Shselasky{ 353321936Shselasky int ret, i; 354321936Shselasky 355321936Shselasky nodes = calloc(sizeof *nodes, connections); 356321936Shselasky if (!nodes) 357321936Shselasky return -ENOMEM; 358321936Shselasky 359321936Shselasky printf("creating id\n"); 360321936Shselasky start_time(STEP_CREATE_ID); 361321936Shselasky for (i = 0; i < connections; i++) { 362321936Shselasky start_perf(&nodes[i], STEP_CREATE_ID); 363321936Shselasky if (dst_addr) { 364321936Shselasky ret = rdma_create_id(channel, &nodes[i].id, &nodes[i], 365321936Shselasky hints.ai_port_space); 366321936Shselasky if (ret) 367321936Shselasky goto err; 368321936Shselasky } 369321936Shselasky end_perf(&nodes[i], STEP_CREATE_ID); 370321936Shselasky } 371321936Shselasky end_time(STEP_CREATE_ID); 372321936Shselasky return 0; 373321936Shselasky 374321936Shselaskyerr: 375321936Shselasky while (--i >= 0) 376321936Shselasky rdma_destroy_id(nodes[i].id); 377321936Shselasky free(nodes); 378321936Shselasky return ret; 379321936Shselasky} 380321936Shselasky 381321936Shselaskystatic void cleanup_nodes(void) 382321936Shselasky{ 383321936Shselasky int i; 384321936Shselasky 385321936Shselasky printf("destroying id\n"); 386321936Shselasky start_time(STEP_DESTROY); 387321936Shselasky for (i = 0; i < connections; i++) { 388321936Shselasky start_perf(&nodes[i], STEP_DESTROY); 389321936Shselasky if (nodes[i].id) 390321936Shselasky rdma_destroy_id(nodes[i].id); 391321936Shselasky end_perf(&nodes[i], STEP_DESTROY); 392321936Shselasky } 393321936Shselasky end_time(STEP_DESTROY); 394321936Shselasky} 395321936Shselasky 396321936Shselaskystatic void *process_events(void *arg) 397321936Shselasky{ 398321936Shselasky struct rdma_cm_event *event; 399321936Shselasky int ret = 0; 400321936Shselasky 401321936Shselasky while (!ret) { 402321936Shselasky ret = rdma_get_cm_event(channel, &event); 403321936Shselasky if (!ret) { 404321936Shselasky cma_handler(event->id, event); 405321936Shselasky } else { 406321936Shselasky perror("failure in rdma_get_cm_event in process_server_events"); 407321936Shselasky ret = errno; 408321936Shselasky } 409321936Shselasky } 410321936Shselasky return NULL; 411321936Shselasky} 412321936Shselasky 413321936Shselaskystatic int run_server(void) 414321936Shselasky{ 415321936Shselasky pthread_t req_thread, disc_thread; 416321936Shselasky struct rdma_cm_id *listen_id; 417321936Shselasky int ret; 418321936Shselasky 419321936Shselasky INIT_LIST(&req_work.list); 420321936Shselasky INIT_LIST(&disc_work.list); 421321936Shselasky ret = pthread_mutex_init(&req_work.lock, NULL); 422321936Shselasky if (ret) { 423321936Shselasky perror("initializing mutex for req work"); 424321936Shselasky return ret; 425321936Shselasky } 426321936Shselasky 427321936Shselasky ret = pthread_mutex_init(&disc_work.lock, NULL); 428321936Shselasky if (ret) { 429321936Shselasky perror("initializing mutex for disc work"); 430321936Shselasky return ret; 431321936Shselasky } 432321936Shselasky 433321936Shselasky ret = pthread_cond_init(&req_work.cond, NULL); 434321936Shselasky if (ret) { 435321936Shselasky perror("initializing cond for req work"); 436321936Shselasky return ret; 437321936Shselasky } 438321936Shselasky 439321936Shselasky ret = pthread_cond_init(&disc_work.cond, NULL); 440321936Shselasky if (ret) { 441321936Shselasky perror("initializing cond for disc work"); 442321936Shselasky return ret; 443321936Shselasky } 444321936Shselasky 445321936Shselasky ret = pthread_create(&req_thread, NULL, req_handler_thread, NULL); 446321936Shselasky if (ret) { 447321936Shselasky perror("failed to create req handler thread"); 448321936Shselasky return ret; 449321936Shselasky } 450321936Shselasky 451321936Shselasky ret = pthread_create(&disc_thread, NULL, disc_handler_thread, NULL); 452321936Shselasky if (ret) { 453321936Shselasky perror("failed to create disconnect handler thread"); 454321936Shselasky return ret; 455321936Shselasky } 456321936Shselasky 457321936Shselasky ret = rdma_create_id(channel, &listen_id, NULL, hints.ai_port_space); 458321936Shselasky if (ret) { 459321936Shselasky perror("listen request failed"); 460321936Shselasky return ret; 461321936Shselasky } 462321936Shselasky 463321936Shselasky ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &rai); 464321936Shselasky if (ret) { 465321936Shselasky printf("getrdmaaddr error: %s\n", gai_strerror(ret)); 466321936Shselasky goto out; 467321936Shselasky } 468321936Shselasky 469321936Shselasky ret = rdma_bind_addr(listen_id, rai->ai_src_addr); 470321936Shselasky if (ret) { 471321936Shselasky perror("bind address failed"); 472321936Shselasky goto out; 473321936Shselasky } 474321936Shselasky 475321936Shselasky ret = rdma_listen(listen_id, 0); 476321936Shselasky if (ret) { 477321936Shselasky perror("failure trying to listen"); 478321936Shselasky goto out; 479321936Shselasky } 480321936Shselasky 481321936Shselasky process_events(NULL); 482321936Shselasky out: 483321936Shselasky rdma_destroy_id(listen_id); 484321936Shselasky return ret; 485321936Shselasky} 486321936Shselasky 487321936Shselaskystatic int run_client(void) 488321936Shselasky{ 489321936Shselasky pthread_t event_thread; 490321936Shselasky int i, ret; 491321936Shselasky 492321936Shselasky ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &rai); 493321936Shselasky if (ret) { 494321936Shselasky printf("getaddrinfo error: %s\n", gai_strerror(ret)); 495321936Shselasky return ret; 496321936Shselasky } 497321936Shselasky 498321936Shselasky conn_param.responder_resources = 1; 499321936Shselasky conn_param.initiator_depth = 1; 500321936Shselasky conn_param.retry_count = retries; 501321936Shselasky conn_param.private_data = rai->ai_connect; 502321936Shselasky conn_param.private_data_len = rai->ai_connect_len; 503321936Shselasky 504321936Shselasky ret = pthread_create(&event_thread, NULL, process_events, NULL); 505321936Shselasky if (ret) { 506321936Shselasky perror("failure creating event thread"); 507321936Shselasky return ret; 508321936Shselasky } 509321936Shselasky 510321936Shselasky if (src_addr) { 511321936Shselasky printf("binding source address\n"); 512321936Shselasky start_time(STEP_BIND); 513321936Shselasky for (i = 0; i < connections; i++) { 514321936Shselasky start_perf(&nodes[i], STEP_BIND); 515321936Shselasky ret = rdma_bind_addr(nodes[i].id, rai->ai_src_addr); 516321936Shselasky if (ret) { 517321936Shselasky perror("failure bind addr"); 518321936Shselasky nodes[i].error = 1; 519321936Shselasky continue; 520321936Shselasky } 521321936Shselasky end_perf(&nodes[i], STEP_BIND); 522321936Shselasky } 523321936Shselasky end_time(STEP_BIND); 524321936Shselasky } 525321936Shselasky 526321936Shselasky printf("resolving address\n"); 527321936Shselasky start_time(STEP_RESOLVE_ADDR); 528321936Shselasky for (i = 0; i < connections; i++) { 529321936Shselasky if (nodes[i].error) 530321936Shselasky continue; 531321936Shselasky nodes[i].retries = retries; 532321936Shselasky start_perf(&nodes[i], STEP_RESOLVE_ADDR); 533321936Shselasky ret = rdma_resolve_addr(nodes[i].id, rai->ai_src_addr, 534321936Shselasky rai->ai_dst_addr, timeout); 535321936Shselasky if (ret) { 536321936Shselasky perror("failure getting addr"); 537321936Shselasky nodes[i].error = 1; 538321936Shselasky continue; 539321936Shselasky } 540321936Shselasky started[STEP_RESOLVE_ADDR]++; 541321936Shselasky } 542321936Shselasky while (started[STEP_RESOLVE_ADDR] != completed[STEP_RESOLVE_ADDR]) sched_yield(); 543321936Shselasky end_time(STEP_RESOLVE_ADDR); 544321936Shselasky 545321936Shselasky printf("resolving route\n"); 546321936Shselasky start_time(STEP_RESOLVE_ROUTE); 547321936Shselasky for (i = 0; i < connections; i++) { 548321936Shselasky if (nodes[i].error) 549321936Shselasky continue; 550321936Shselasky nodes[i].retries = retries; 551321936Shselasky start_perf(&nodes[i], STEP_RESOLVE_ROUTE); 552321936Shselasky ret = rdma_resolve_route(nodes[i].id, timeout); 553321936Shselasky if (ret) { 554321936Shselasky perror("failure resolving route"); 555321936Shselasky nodes[i].error = 1; 556321936Shselasky continue; 557321936Shselasky } 558321936Shselasky started[STEP_RESOLVE_ROUTE]++; 559321936Shselasky } 560321936Shselasky while (started[STEP_RESOLVE_ROUTE] != completed[STEP_RESOLVE_ROUTE]) sched_yield(); 561321936Shselasky end_time(STEP_RESOLVE_ROUTE); 562321936Shselasky 563321936Shselasky printf("creating qp\n"); 564321936Shselasky start_time(STEP_CREATE_QP); 565321936Shselasky for (i = 0; i < connections; i++) { 566321936Shselasky if (nodes[i].error) 567321936Shselasky continue; 568321936Shselasky start_perf(&nodes[i], STEP_CREATE_QP); 569321936Shselasky ret = rdma_create_qp(nodes[i].id, NULL, &init_qp_attr); 570321936Shselasky if (ret) { 571321936Shselasky perror("failure creating qp"); 572321936Shselasky nodes[i].error = 1; 573321936Shselasky continue; 574321936Shselasky } 575321936Shselasky end_perf(&nodes[i], STEP_CREATE_QP); 576321936Shselasky } 577321936Shselasky end_time(STEP_CREATE_QP); 578321936Shselasky 579321936Shselasky printf("connecting\n"); 580321936Shselasky start_time(STEP_CONNECT); 581321936Shselasky for (i = 0; i < connections; i++) { 582321936Shselasky if (nodes[i].error) 583321936Shselasky continue; 584321936Shselasky start_perf(&nodes[i], STEP_CONNECT); 585321936Shselasky ret = rdma_connect(nodes[i].id, &conn_param); 586321936Shselasky if (ret) { 587321936Shselasky perror("failure rconnecting"); 588321936Shselasky nodes[i].error = 1; 589321936Shselasky continue; 590321936Shselasky } 591321936Shselasky started[STEP_CONNECT]++; 592321936Shselasky } 593321936Shselasky while (started[STEP_CONNECT] != completed[STEP_CONNECT]) sched_yield(); 594321936Shselasky end_time(STEP_CONNECT); 595321936Shselasky 596321936Shselasky printf("disconnecting\n"); 597321936Shselasky start_time(STEP_DISCONNECT); 598321936Shselasky for (i = 0; i < connections; i++) { 599321936Shselasky if (nodes[i].error) 600321936Shselasky continue; 601321936Shselasky start_perf(&nodes[i], STEP_DISCONNECT); 602321936Shselasky rdma_disconnect(nodes[i].id); 603321936Shselasky started[STEP_DISCONNECT]++; 604321936Shselasky } 605321936Shselasky while (started[STEP_DISCONNECT] != completed[STEP_DISCONNECT]) sched_yield(); 606321936Shselasky end_time(STEP_DISCONNECT); 607321936Shselasky 608321936Shselasky return ret; 609321936Shselasky} 610321936Shselasky 611321936Shselaskyint main(int argc, char **argv) 612321936Shselasky{ 613321936Shselasky int op, ret; 614321936Shselasky 615321936Shselasky hints.ai_port_space = RDMA_PS_TCP; 616321936Shselasky hints.ai_qp_type = IBV_QPT_RC; 617321936Shselasky while ((op = getopt(argc, argv, "s:b:c:p:r:t:")) != -1) { 618321936Shselasky switch (op) { 619321936Shselasky case 's': 620321936Shselasky dst_addr = optarg; 621321936Shselasky break; 622321936Shselasky case 'b': 623321936Shselasky src_addr = optarg; 624321936Shselasky break; 625321936Shselasky case 'c': 626321936Shselasky connections = atoi(optarg); 627321936Shselasky break; 628321936Shselasky case 'p': 629321936Shselasky port = optarg; 630321936Shselasky break; 631321936Shselasky case 'r': 632321936Shselasky retries = atoi(optarg); 633321936Shselasky break; 634321936Shselasky case 't': 635321936Shselasky timeout = atoi(optarg); 636321936Shselasky break; 637321936Shselasky default: 638321936Shselasky printf("usage: %s\n", argv[0]); 639321936Shselasky printf("\t[-s server_address]\n"); 640321936Shselasky printf("\t[-b bind_address]\n"); 641321936Shselasky printf("\t[-c connections]\n"); 642321936Shselasky printf("\t[-p port_number]\n"); 643321936Shselasky printf("\t[-r retries]\n"); 644321936Shselasky printf("\t[-t timeout_ms]\n"); 645321936Shselasky exit(1); 646321936Shselasky } 647321936Shselasky } 648321936Shselasky 649321936Shselasky init_qp_attr.cap.max_send_wr = 1; 650321936Shselasky init_qp_attr.cap.max_recv_wr = 1; 651321936Shselasky init_qp_attr.cap.max_send_sge = 1; 652321936Shselasky init_qp_attr.cap.max_recv_sge = 1; 653321936Shselasky init_qp_attr.qp_type = IBV_QPT_RC; 654321936Shselasky 655321936Shselasky channel = rdma_create_event_channel(); 656321936Shselasky if (!channel) { 657321936Shselasky printf("failed to create event channel\n"); 658321936Shselasky exit(1); 659321936Shselasky } 660321936Shselasky 661321936Shselasky if (dst_addr) { 662321936Shselasky alloc_nodes(); 663321936Shselasky ret = run_client(); 664321936Shselasky } else { 665321936Shselasky hints.ai_flags |= RAI_PASSIVE; 666321936Shselasky ret = run_server(); 667321936Shselasky } 668321936Shselasky 669321936Shselasky cleanup_nodes(); 670321936Shselasky rdma_destroy_event_channel(channel); 671321936Shselasky if (rai) 672321936Shselasky rdma_freeaddrinfo(rai); 673321936Shselasky 674321936Shselasky show_perf(); 675321936Shselasky free(nodes); 676321936Shselasky return ret; 677321936Shselasky} 678