1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Topspin Communications. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#if HAVE_CONFIG_H 34219820Sjeff# include <config.h> 35219820Sjeff#endif /* HAVE_CONFIG_H */ 36219820Sjeff 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <stdlib.h> 39219820Sjeff#include <unistd.h> 40219820Sjeff#include <string.h> 41219820Sjeff#include <sys/types.h> 42219820Sjeff#include <sys/socket.h> 43219820Sjeff#include <sys/time.h> 44219820Sjeff#include <netdb.h> 45219820Sjeff#include <getopt.h> 46219820Sjeff#include <arpa/inet.h> 47219820Sjeff#include <time.h> 48219820Sjeff 49219820Sjeff#include "pingpong.h" 50219820Sjeff 51219820Sjeffenum { 52219820Sjeff PINGPONG_RECV_WRID = 1, 53219820Sjeff PINGPONG_SEND_WRID = 2, 54219820Sjeff}; 55219820Sjeff 56219820Sjeffstatic int page_size; 57219820Sjeff 58219820Sjeffstruct pingpong_context { 59219820Sjeff struct ibv_context *context; 60219820Sjeff struct ibv_comp_channel *channel; 61219820Sjeff struct ibv_pd *pd; 62219820Sjeff struct ibv_mr *mr; 63219820Sjeff struct ibv_cq *cq; 64219820Sjeff struct ibv_qp *qp; 65219820Sjeff void *buf; 66219820Sjeff int size; 67219820Sjeff int rx_depth; 68219820Sjeff int pending; 69219820Sjeff struct ibv_port_attr portinfo; 70219820Sjeff}; 71219820Sjeff 72219820Sjeffstruct pingpong_dest { 73219820Sjeff int lid; 74219820Sjeff int qpn; 75219820Sjeff int psn; 76219820Sjeff union ibv_gid gid; 77219820Sjeff}; 78219820Sjeff 79219820Sjeffstatic int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 80219820Sjeff enum ibv_mtu mtu, int sl, 81219820Sjeff struct pingpong_dest *dest, int sgid_idx) 82219820Sjeff{ 83219820Sjeff struct ibv_qp_attr attr = { 84219820Sjeff .qp_state = IBV_QPS_RTR, 85219820Sjeff .path_mtu = mtu, 86219820Sjeff .dest_qp_num = dest->qpn, 87219820Sjeff .rq_psn = dest->psn, 88219820Sjeff .ah_attr = { 89219820Sjeff .is_global = 0, 90219820Sjeff .dlid = dest->lid, 91219820Sjeff .sl = sl, 92219820Sjeff .src_path_bits = 0, 93219820Sjeff .port_num = port 94219820Sjeff } 95219820Sjeff }; 96219820Sjeff 97219820Sjeff if (dest->gid.global.interface_id) { 98219820Sjeff attr.ah_attr.is_global = 1; 99219820Sjeff attr.ah_attr.grh.hop_limit = 1; 100219820Sjeff attr.ah_attr.grh.dgid = dest->gid; 101219820Sjeff attr.ah_attr.grh.sgid_index = sgid_idx; 102219820Sjeff } 103219820Sjeff 104219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 105219820Sjeff IBV_QP_STATE | 106219820Sjeff IBV_QP_AV | 107219820Sjeff IBV_QP_PATH_MTU | 108219820Sjeff IBV_QP_DEST_QPN | 109219820Sjeff IBV_QP_RQ_PSN)) { 110219820Sjeff fprintf(stderr, "Failed to modify QP to RTR\n"); 111219820Sjeff return 1; 112219820Sjeff } 113219820Sjeff 114219820Sjeff attr.qp_state = IBV_QPS_RTS; 115219820Sjeff attr.sq_psn = my_psn; 116219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 117219820Sjeff IBV_QP_STATE | 118219820Sjeff IBV_QP_SQ_PSN)) { 119219820Sjeff fprintf(stderr, "Failed to modify QP to RTS\n"); 120219820Sjeff return 1; 121219820Sjeff } 122219820Sjeff 123219820Sjeff return 0; 124219820Sjeff} 125219820Sjeff 126219820Sjeffstatic struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, 127219820Sjeff const struct pingpong_dest *my_dest) 128219820Sjeff{ 129219820Sjeff struct addrinfo *res, *t; 130219820Sjeff struct addrinfo hints = { 131219820Sjeff .ai_family = AF_INET, 132219820Sjeff .ai_socktype = SOCK_STREAM 133219820Sjeff }; 134219820Sjeff char *service; 135219820Sjeff char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 136219820Sjeff int n; 137219820Sjeff int sockfd = -1; 138219820Sjeff struct pingpong_dest *rem_dest = NULL; 139219820Sjeff char gid[33]; 140219820Sjeff 141219820Sjeff if (asprintf(&service, "%d", port) < 0) 142219820Sjeff return NULL; 143219820Sjeff 144219820Sjeff n = getaddrinfo(servername, service, &hints, &res); 145219820Sjeff 146219820Sjeff if (n < 0) { 147219820Sjeff fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); 148219820Sjeff free(service); 149219820Sjeff return NULL; 150219820Sjeff } 151219820Sjeff 152219820Sjeff for (t = res; t; t = t->ai_next) { 153219820Sjeff sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 154219820Sjeff if (sockfd >= 0) { 155219820Sjeff if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) 156219820Sjeff break; 157219820Sjeff close(sockfd); 158219820Sjeff sockfd = -1; 159219820Sjeff } 160219820Sjeff } 161219820Sjeff 162219820Sjeff freeaddrinfo(res); 163219820Sjeff free(service); 164219820Sjeff 165219820Sjeff if (sockfd < 0) { 166219820Sjeff fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); 167219820Sjeff return NULL; 168219820Sjeff } 169219820Sjeff 170219820Sjeff gid_to_wire_gid(&my_dest->gid, gid); 171219820Sjeff sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 172219820Sjeff if (write(sockfd, msg, sizeof msg) != sizeof msg) { 173219820Sjeff fprintf(stderr, "Couldn't send local address\n"); 174219820Sjeff goto out; 175219820Sjeff } 176219820Sjeff 177219820Sjeff if (read(sockfd, msg, sizeof msg) != sizeof msg) { 178219820Sjeff perror("client read"); 179219820Sjeff fprintf(stderr, "Couldn't read remote address\n"); 180219820Sjeff goto out; 181219820Sjeff } 182219820Sjeff 183219820Sjeff write(sockfd, "done", sizeof "done"); 184219820Sjeff 185219820Sjeff rem_dest = malloc(sizeof *rem_dest); 186219820Sjeff if (!rem_dest) 187219820Sjeff goto out; 188219820Sjeff 189219820Sjeff sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 190219820Sjeff wire_gid_to_gid(gid, &rem_dest->gid); 191219820Sjeff 192219820Sjeffout: 193219820Sjeff close(sockfd); 194219820Sjeff return rem_dest; 195219820Sjeff} 196219820Sjeff 197219820Sjeffstatic struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 198219820Sjeff int ib_port, enum ibv_mtu mtu, 199219820Sjeff int port, int sl, 200219820Sjeff const struct pingpong_dest *my_dest, 201219820Sjeff int sgid_idx) 202219820Sjeff{ 203219820Sjeff struct addrinfo *res, *t; 204219820Sjeff struct addrinfo hints = { 205219820Sjeff .ai_flags = AI_PASSIVE, 206219820Sjeff .ai_family = AF_INET, 207219820Sjeff .ai_socktype = SOCK_STREAM 208219820Sjeff }; 209219820Sjeff char *service; 210219820Sjeff char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 211219820Sjeff int n; 212219820Sjeff int sockfd = -1, connfd; 213219820Sjeff struct pingpong_dest *rem_dest = NULL; 214219820Sjeff char gid[33]; 215219820Sjeff 216219820Sjeff if (asprintf(&service, "%d", port) < 0) 217219820Sjeff return NULL; 218219820Sjeff 219219820Sjeff n = getaddrinfo(NULL, service, &hints, &res); 220219820Sjeff 221219820Sjeff if (n < 0) { 222219820Sjeff fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); 223219820Sjeff free(service); 224219820Sjeff return NULL; 225219820Sjeff } 226219820Sjeff 227219820Sjeff for (t = res; t; t = t->ai_next) { 228219820Sjeff sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 229219820Sjeff if (sockfd >= 0) { 230219820Sjeff n = 1; 231219820Sjeff 232219820Sjeff setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); 233219820Sjeff 234219820Sjeff if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) 235219820Sjeff break; 236219820Sjeff close(sockfd); 237219820Sjeff sockfd = -1; 238219820Sjeff } 239219820Sjeff } 240219820Sjeff 241219820Sjeff freeaddrinfo(res); 242219820Sjeff free(service); 243219820Sjeff 244219820Sjeff if (sockfd < 0) { 245219820Sjeff fprintf(stderr, "Couldn't listen to port %d\n", port); 246219820Sjeff return NULL; 247219820Sjeff } 248219820Sjeff 249219820Sjeff listen(sockfd, 1); 250219820Sjeff connfd = accept(sockfd, NULL, 0); 251219820Sjeff close(sockfd); 252219820Sjeff if (connfd < 0) { 253219820Sjeff fprintf(stderr, "accept() failed\n"); 254219820Sjeff return NULL; 255219820Sjeff } 256219820Sjeff 257219820Sjeff n = read(connfd, msg, sizeof msg); 258219820Sjeff if (n != sizeof msg) { 259219820Sjeff perror("server read"); 260219820Sjeff fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); 261219820Sjeff goto out; 262219820Sjeff } 263219820Sjeff 264219820Sjeff rem_dest = malloc(sizeof *rem_dest); 265219820Sjeff if (!rem_dest) 266219820Sjeff goto out; 267219820Sjeff 268219820Sjeff sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 269219820Sjeff wire_gid_to_gid(gid, &rem_dest->gid); 270219820Sjeff 271219820Sjeff if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 272219820Sjeff fprintf(stderr, "Couldn't connect to remote QP\n"); 273219820Sjeff free(rem_dest); 274219820Sjeff rem_dest = NULL; 275219820Sjeff goto out; 276219820Sjeff } 277219820Sjeff 278219820Sjeff gid_to_wire_gid(&my_dest->gid, gid); 279219820Sjeff sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 280219820Sjeff if (write(connfd, msg, sizeof msg) != sizeof msg) { 281219820Sjeff fprintf(stderr, "Couldn't send local address\n"); 282219820Sjeff free(rem_dest); 283219820Sjeff rem_dest = NULL; 284219820Sjeff goto out; 285219820Sjeff } 286219820Sjeff 287219820Sjeff read(connfd, msg, sizeof msg); 288219820Sjeff 289219820Sjeffout: 290219820Sjeff close(connfd); 291219820Sjeff return rem_dest; 292219820Sjeff} 293219820Sjeff 294219820Sjeffstatic struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 295219820Sjeff int rx_depth, int port, 296219820Sjeff int use_event) 297219820Sjeff{ 298219820Sjeff struct pingpong_context *ctx; 299219820Sjeff 300219820Sjeff ctx = calloc(1, sizeof *ctx); 301219820Sjeff if (!ctx) 302219820Sjeff return NULL; 303219820Sjeff 304219820Sjeff ctx->size = size; 305219820Sjeff ctx->rx_depth = rx_depth; 306219820Sjeff 307219820Sjeff ctx->buf = malloc(roundup(size, page_size)); 308219820Sjeff if (!ctx->buf) { 309219820Sjeff fprintf(stderr, "Couldn't allocate work buf.\n"); 310219820Sjeff return NULL; 311219820Sjeff } 312219820Sjeff 313219820Sjeff memset(ctx->buf, 0, size); 314219820Sjeff 315219820Sjeff ctx->context = ibv_open_device(ib_dev); 316219820Sjeff if (!ctx->context) { 317219820Sjeff fprintf(stderr, "Couldn't get context for %s\n", 318219820Sjeff ibv_get_device_name(ib_dev)); 319219820Sjeff return NULL; 320219820Sjeff } 321219820Sjeff 322219820Sjeff if (use_event) { 323219820Sjeff ctx->channel = ibv_create_comp_channel(ctx->context); 324219820Sjeff if (!ctx->channel) { 325219820Sjeff fprintf(stderr, "Couldn't create completion channel\n"); 326219820Sjeff return NULL; 327219820Sjeff } 328219820Sjeff } else 329219820Sjeff ctx->channel = NULL; 330219820Sjeff 331219820Sjeff ctx->pd = ibv_alloc_pd(ctx->context); 332219820Sjeff if (!ctx->pd) { 333219820Sjeff fprintf(stderr, "Couldn't allocate PD\n"); 334219820Sjeff return NULL; 335219820Sjeff } 336219820Sjeff 337219820Sjeff ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); 338219820Sjeff if (!ctx->mr) { 339219820Sjeff fprintf(stderr, "Couldn't register MR\n"); 340219820Sjeff return NULL; 341219820Sjeff } 342219820Sjeff 343219820Sjeff ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, 344219820Sjeff ctx->channel, 0); 345219820Sjeff if (!ctx->cq) { 346219820Sjeff fprintf(stderr, "Couldn't create CQ\n"); 347219820Sjeff return NULL; 348219820Sjeff } 349219820Sjeff 350219820Sjeff { 351219820Sjeff struct ibv_qp_init_attr attr = { 352219820Sjeff .send_cq = ctx->cq, 353219820Sjeff .recv_cq = ctx->cq, 354219820Sjeff .cap = { 355219820Sjeff .max_send_wr = 1, 356219820Sjeff .max_recv_wr = rx_depth, 357219820Sjeff .max_send_sge = 1, 358219820Sjeff .max_recv_sge = 1 359219820Sjeff }, 360219820Sjeff .qp_type = IBV_QPT_UC 361219820Sjeff }; 362219820Sjeff 363219820Sjeff ctx->qp = ibv_create_qp(ctx->pd, &attr); 364219820Sjeff if (!ctx->qp) { 365219820Sjeff fprintf(stderr, "Couldn't create QP\n"); 366219820Sjeff return NULL; 367219820Sjeff } 368219820Sjeff } 369219820Sjeff 370219820Sjeff { 371219820Sjeff struct ibv_qp_attr attr = { 372219820Sjeff .qp_state = IBV_QPS_INIT, 373219820Sjeff .pkey_index = 0, 374219820Sjeff .port_num = port, 375219820Sjeff .qp_access_flags = 0 376219820Sjeff }; 377219820Sjeff 378219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 379219820Sjeff IBV_QP_STATE | 380219820Sjeff IBV_QP_PKEY_INDEX | 381219820Sjeff IBV_QP_PORT | 382219820Sjeff IBV_QP_ACCESS_FLAGS)) { 383219820Sjeff fprintf(stderr, "Failed to modify QP to INIT\n"); 384219820Sjeff return NULL; 385219820Sjeff } 386219820Sjeff } 387219820Sjeff 388219820Sjeff return ctx; 389219820Sjeff} 390219820Sjeff 391219820Sjeffint pp_close_ctx(struct pingpong_context *ctx) 392219820Sjeff{ 393219820Sjeff if (ibv_destroy_qp(ctx->qp)) { 394219820Sjeff fprintf(stderr, "Couldn't destroy QP\n"); 395219820Sjeff return 1; 396219820Sjeff } 397219820Sjeff 398219820Sjeff if (ibv_destroy_cq(ctx->cq)) { 399219820Sjeff fprintf(stderr, "Couldn't destroy CQ\n"); 400219820Sjeff return 1; 401219820Sjeff } 402219820Sjeff 403219820Sjeff if (ibv_dereg_mr(ctx->mr)) { 404219820Sjeff fprintf(stderr, "Couldn't deregister MR\n"); 405219820Sjeff return 1; 406219820Sjeff } 407219820Sjeff 408219820Sjeff if (ibv_dealloc_pd(ctx->pd)) { 409219820Sjeff fprintf(stderr, "Couldn't deallocate PD\n"); 410219820Sjeff return 1; 411219820Sjeff } 412219820Sjeff 413219820Sjeff if (ctx->channel) { 414219820Sjeff if (ibv_destroy_comp_channel(ctx->channel)) { 415219820Sjeff fprintf(stderr, "Couldn't destroy completion channel\n"); 416219820Sjeff return 1; 417219820Sjeff } 418219820Sjeff } 419219820Sjeff 420219820Sjeff if (ibv_close_device(ctx->context)) { 421219820Sjeff fprintf(stderr, "Couldn't release context\n"); 422219820Sjeff return 1; 423219820Sjeff } 424219820Sjeff 425219820Sjeff free(ctx->buf); 426219820Sjeff free(ctx); 427219820Sjeff 428219820Sjeff return 0; 429219820Sjeff} 430219820Sjeff 431219820Sjeffstatic int pp_post_recv(struct pingpong_context *ctx, int n) 432219820Sjeff{ 433219820Sjeff struct ibv_sge list = { 434219820Sjeff .addr = (uintptr_t) ctx->buf, 435219820Sjeff .length = ctx->size, 436219820Sjeff .lkey = ctx->mr->lkey 437219820Sjeff }; 438219820Sjeff struct ibv_recv_wr wr = { 439219820Sjeff .wr_id = PINGPONG_RECV_WRID, 440219820Sjeff .sg_list = &list, 441219820Sjeff .num_sge = 1, 442219820Sjeff }; 443219820Sjeff struct ibv_recv_wr *bad_wr; 444219820Sjeff int i; 445219820Sjeff 446219820Sjeff for (i = 0; i < n; ++i) 447219820Sjeff if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 448219820Sjeff break; 449219820Sjeff 450219820Sjeff return i; 451219820Sjeff} 452219820Sjeff 453219820Sjeffstatic int pp_post_send(struct pingpong_context *ctx) 454219820Sjeff{ 455219820Sjeff struct ibv_sge list = { 456219820Sjeff .addr = (uintptr_t) ctx->buf, 457219820Sjeff .length = ctx->size, 458219820Sjeff .lkey = ctx->mr->lkey 459219820Sjeff }; 460219820Sjeff struct ibv_send_wr wr = { 461219820Sjeff .wr_id = PINGPONG_SEND_WRID, 462219820Sjeff .sg_list = &list, 463219820Sjeff .num_sge = 1, 464219820Sjeff .opcode = IBV_WR_SEND, 465219820Sjeff .send_flags = IBV_SEND_SIGNALED, 466219820Sjeff }; 467219820Sjeff struct ibv_send_wr *bad_wr; 468219820Sjeff 469219820Sjeff return ibv_post_send(ctx->qp, &wr, &bad_wr); 470219820Sjeff} 471219820Sjeff 472219820Sjeffstatic void usage(const char *argv0) 473219820Sjeff{ 474219820Sjeff printf("Usage:\n"); 475219820Sjeff printf(" %s start a server and wait for connection\n", argv0); 476219820Sjeff printf(" %s <host> connect to server at <host>\n", argv0); 477219820Sjeff printf("\n"); 478219820Sjeff printf("Options:\n"); 479219820Sjeff printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 480219820Sjeff printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 481219820Sjeff printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 482219820Sjeff printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); 483219820Sjeff printf(" -m, --mtu=<size> path MTU (default 1024)\n"); 484219820Sjeff printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 485219820Sjeff printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 486219820Sjeff printf(" -l, --sl=<sl> service level value\n"); 487219820Sjeff printf(" -e, --events sleep on CQ events (default poll)\n"); 488219820Sjeff printf(" -g, --gid-idx=<gid index> local port gid index\n"); 489219820Sjeff} 490219820Sjeff 491219820Sjeffint main(int argc, char *argv[]) 492219820Sjeff{ 493219820Sjeff struct ibv_device **dev_list; 494219820Sjeff struct ibv_device *ib_dev; 495219820Sjeff struct pingpong_context *ctx; 496219820Sjeff struct pingpong_dest my_dest; 497219820Sjeff struct pingpong_dest *rem_dest; 498219820Sjeff struct timeval start, end; 499219820Sjeff char *ib_devname = NULL; 500219820Sjeff char *servername = NULL; 501219820Sjeff int port = 18515; 502219820Sjeff int ib_port = 1; 503219820Sjeff int size = 4096; 504219820Sjeff enum ibv_mtu mtu = IBV_MTU_1024; 505219820Sjeff int rx_depth = 500; 506219820Sjeff int iters = 1000; 507219820Sjeff int use_event = 0; 508219820Sjeff int routs; 509219820Sjeff int rcnt, scnt; 510219820Sjeff int num_cq_events = 0; 511219820Sjeff int sl = 0; 512219820Sjeff int gidx = -1; 513219820Sjeff char gid[33]; 514219820Sjeff 515219820Sjeff srand48(getpid() * time(NULL)); 516219820Sjeff 517219820Sjeff while (1) { 518219820Sjeff int c; 519219820Sjeff 520219820Sjeff static struct option long_options[] = { 521219820Sjeff { .name = "port", .has_arg = 1, .val = 'p' }, 522219820Sjeff { .name = "ib-dev", .has_arg = 1, .val = 'd' }, 523219820Sjeff { .name = "ib-port", .has_arg = 1, .val = 'i' }, 524219820Sjeff { .name = "size", .has_arg = 1, .val = 's' }, 525219820Sjeff { .name = "mtu", .has_arg = 1, .val = 'm' }, 526219820Sjeff { .name = "rx-depth", .has_arg = 1, .val = 'r' }, 527219820Sjeff { .name = "iters", .has_arg = 1, .val = 'n' }, 528219820Sjeff { .name = "sl", .has_arg = 1, .val = 'l' }, 529219820Sjeff { .name = "events", .has_arg = 0, .val = 'e' }, 530219820Sjeff { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 531219820Sjeff { 0 } 532219820Sjeff }; 533219820Sjeff 534219820Sjeff c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 535219820Sjeff if (c == -1) 536219820Sjeff break; 537219820Sjeff 538219820Sjeff switch (c) { 539219820Sjeff case 'p': 540219820Sjeff port = strtol(optarg, NULL, 0); 541219820Sjeff if (port < 0 || port > 65535) { 542219820Sjeff usage(argv[0]); 543219820Sjeff return 1; 544219820Sjeff } 545219820Sjeff break; 546219820Sjeff 547219820Sjeff case 'd': 548219820Sjeff ib_devname = strdup(optarg); 549219820Sjeff break; 550219820Sjeff 551219820Sjeff case 'i': 552219820Sjeff ib_port = strtol(optarg, NULL, 0); 553219820Sjeff if (ib_port < 0) { 554219820Sjeff usage(argv[0]); 555219820Sjeff return 1; 556219820Sjeff } 557219820Sjeff break; 558219820Sjeff 559219820Sjeff case 's': 560219820Sjeff size = strtol(optarg, NULL, 0); 561219820Sjeff break; 562219820Sjeff 563219820Sjeff case 'm': 564219820Sjeff mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); 565219820Sjeff if (mtu < 0) { 566219820Sjeff usage(argv[0]); 567219820Sjeff return 1; 568219820Sjeff } 569219820Sjeff break; 570219820Sjeff 571219820Sjeff case 'r': 572219820Sjeff rx_depth = strtol(optarg, NULL, 0); 573219820Sjeff break; 574219820Sjeff 575219820Sjeff case 'n': 576219820Sjeff iters = strtol(optarg, NULL, 0); 577219820Sjeff break; 578219820Sjeff 579219820Sjeff case 'l': 580219820Sjeff sl = strtol(optarg, NULL, 0); 581219820Sjeff break; 582219820Sjeff 583219820Sjeff case 'e': 584219820Sjeff ++use_event; 585219820Sjeff break; 586219820Sjeff 587219820Sjeff case 'g': 588219820Sjeff gidx = strtol(optarg, NULL, 0); 589219820Sjeff break; 590219820Sjeff 591219820Sjeff default: 592219820Sjeff usage(argv[0]); 593219820Sjeff return 1; 594219820Sjeff } 595219820Sjeff } 596219820Sjeff 597219820Sjeff if (optind == argc - 1) 598219820Sjeff servername = strdup(argv[optind]); 599219820Sjeff else if (optind < argc) { 600219820Sjeff usage(argv[0]); 601219820Sjeff return 1; 602219820Sjeff } 603219820Sjeff 604219820Sjeff page_size = sysconf(_SC_PAGESIZE); 605219820Sjeff 606219820Sjeff dev_list = ibv_get_device_list(NULL); 607219820Sjeff if (!dev_list) { 608219820Sjeff perror("Failed to get IB devices list"); 609219820Sjeff return 1; 610219820Sjeff } 611219820Sjeff 612219820Sjeff if (!ib_devname) { 613219820Sjeff ib_dev = *dev_list; 614219820Sjeff if (!ib_dev) { 615219820Sjeff fprintf(stderr, "No IB devices found\n"); 616219820Sjeff return 1; 617219820Sjeff } 618219820Sjeff } else { 619219820Sjeff int i; 620219820Sjeff for (i = 0; dev_list[i]; ++i) 621219820Sjeff if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) 622219820Sjeff break; 623219820Sjeff ib_dev = dev_list[i]; 624219820Sjeff if (!ib_dev) { 625219820Sjeff fprintf(stderr, "IB device %s not found\n", ib_devname); 626219820Sjeff return 1; 627219820Sjeff } 628219820Sjeff } 629219820Sjeff 630219820Sjeff ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 631219820Sjeff if (!ctx) 632219820Sjeff return 1; 633219820Sjeff 634219820Sjeff routs = pp_post_recv(ctx, ctx->rx_depth); 635219820Sjeff if (routs < ctx->rx_depth) { 636219820Sjeff fprintf(stderr, "Couldn't post receive (%d)\n", routs); 637219820Sjeff return 1; 638219820Sjeff } 639219820Sjeff 640219820Sjeff if (use_event) 641219820Sjeff if (ibv_req_notify_cq(ctx->cq, 0)) { 642219820Sjeff fprintf(stderr, "Couldn't request CQ notification\n"); 643219820Sjeff return 1; 644219820Sjeff } 645219820Sjeff 646219820Sjeff if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 647219820Sjeff fprintf(stderr, "Couldn't get port info\n"); 648219820Sjeff return 1; 649219820Sjeff } 650219820Sjeff 651219820Sjeff my_dest.lid = ctx->portinfo.lid; 652219820Sjeff if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 653219820Sjeff fprintf(stderr, "Couldn't get local LID\n"); 654219820Sjeff return 1; 655219820Sjeff } 656219820Sjeff 657219820Sjeff if (gidx >= 0) { 658219820Sjeff if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 659219820Sjeff fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 660219820Sjeff return 1; 661219820Sjeff } 662219820Sjeff } else 663219820Sjeff memset(&my_dest.gid, 0, sizeof my_dest.gid); 664219820Sjeff 665219820Sjeff my_dest.qpn = ctx->qp->qp_num; 666219820Sjeff my_dest.psn = lrand48() & 0xffffff; 667219820Sjeff inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 668219820Sjeff printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 669219820Sjeff my_dest.lid, my_dest.qpn, my_dest.psn, gid); 670219820Sjeff 671219820Sjeff if (servername) 672219820Sjeff rem_dest = pp_client_exch_dest(servername, port, &my_dest); 673219820Sjeff else 674219820Sjeff rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 675219820Sjeff 676219820Sjeff if (!rem_dest) 677219820Sjeff return 1; 678219820Sjeff 679219820Sjeff inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 680219820Sjeff printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 681219820Sjeff rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 682219820Sjeff 683219820Sjeff if (servername) 684219820Sjeff if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 685219820Sjeff return 1; 686219820Sjeff 687219820Sjeff ctx->pending = PINGPONG_RECV_WRID; 688219820Sjeff 689219820Sjeff if (servername) { 690219820Sjeff if (pp_post_send(ctx)) { 691219820Sjeff fprintf(stderr, "Couldn't post send\n"); 692219820Sjeff return 1; 693219820Sjeff } 694219820Sjeff ctx->pending |= PINGPONG_SEND_WRID; 695219820Sjeff } 696219820Sjeff 697219820Sjeff if (gettimeofday(&start, NULL)) { 698219820Sjeff perror("gettimeofday"); 699219820Sjeff return 1; 700219820Sjeff } 701219820Sjeff 702219820Sjeff rcnt = scnt = 0; 703219820Sjeff while (rcnt < iters || scnt < iters) { 704219820Sjeff if (use_event) { 705219820Sjeff struct ibv_cq *ev_cq; 706219820Sjeff void *ev_ctx; 707219820Sjeff 708219820Sjeff if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 709219820Sjeff fprintf(stderr, "Failed to get cq_event\n"); 710219820Sjeff return 1; 711219820Sjeff } 712219820Sjeff 713219820Sjeff ++num_cq_events; 714219820Sjeff 715219820Sjeff if (ev_cq != ctx->cq) { 716219820Sjeff fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); 717219820Sjeff return 1; 718219820Sjeff } 719219820Sjeff 720219820Sjeff if (ibv_req_notify_cq(ctx->cq, 0)) { 721219820Sjeff fprintf(stderr, "Couldn't request CQ notification\n"); 722219820Sjeff return 1; 723219820Sjeff } 724219820Sjeff } 725219820Sjeff 726219820Sjeff { 727219820Sjeff struct ibv_wc wc[2]; 728219820Sjeff int ne, i; 729219820Sjeff 730219820Sjeff do { 731219820Sjeff ne = ibv_poll_cq(ctx->cq, 2, wc); 732219820Sjeff if (ne < 0) { 733219820Sjeff fprintf(stderr, "poll CQ failed %d\n", ne); 734219820Sjeff return 1; 735219820Sjeff } 736219820Sjeff } while (!use_event && ne < 1); 737219820Sjeff 738219820Sjeff for (i = 0; i < ne; ++i) { 739219820Sjeff if (wc[i].status != IBV_WC_SUCCESS) { 740219820Sjeff fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", 741219820Sjeff ibv_wc_status_str(wc[i].status), 742219820Sjeff wc[i].status, (int) wc[i].wr_id); 743219820Sjeff return 1; 744219820Sjeff } 745219820Sjeff 746219820Sjeff switch ((int) wc[i].wr_id) { 747219820Sjeff case PINGPONG_SEND_WRID: 748219820Sjeff ++scnt; 749219820Sjeff break; 750219820Sjeff 751219820Sjeff case PINGPONG_RECV_WRID: 752219820Sjeff if (--routs <= 1) { 753219820Sjeff routs += pp_post_recv(ctx, ctx->rx_depth - routs); 754219820Sjeff if (routs < ctx->rx_depth) { 755219820Sjeff fprintf(stderr, 756219820Sjeff "Couldn't post receive (%d)\n", 757219820Sjeff routs); 758219820Sjeff return 1; 759219820Sjeff } 760219820Sjeff } 761219820Sjeff 762219820Sjeff ++rcnt; 763219820Sjeff break; 764219820Sjeff 765219820Sjeff default: 766219820Sjeff fprintf(stderr, "Completion for unknown wr_id %d\n", 767219820Sjeff (int) wc[i].wr_id); 768219820Sjeff return 1; 769219820Sjeff } 770219820Sjeff 771219820Sjeff ctx->pending &= ~(int) wc[i].wr_id; 772219820Sjeff if (scnt < iters && !ctx->pending) { 773219820Sjeff if (pp_post_send(ctx)) { 774219820Sjeff fprintf(stderr, "Couldn't post send\n"); 775219820Sjeff return 1; 776219820Sjeff } 777219820Sjeff ctx->pending = PINGPONG_RECV_WRID | 778219820Sjeff PINGPONG_SEND_WRID; 779219820Sjeff } 780219820Sjeff } 781219820Sjeff } 782219820Sjeff } 783219820Sjeff 784219820Sjeff if (gettimeofday(&end, NULL)) { 785219820Sjeff perror("gettimeofday"); 786219820Sjeff return 1; 787219820Sjeff } 788219820Sjeff 789219820Sjeff { 790219820Sjeff float usec = (end.tv_sec - start.tv_sec) * 1000000 + 791219820Sjeff (end.tv_usec - start.tv_usec); 792219820Sjeff long long bytes = (long long) size * iters * 2; 793219820Sjeff 794219820Sjeff printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", 795219820Sjeff bytes, usec / 1000000., bytes * 8. / usec); 796219820Sjeff printf("%d iters in %.2f seconds = %.2f usec/iter\n", 797219820Sjeff iters, usec / 1000000., usec / iters); 798219820Sjeff } 799219820Sjeff 800219820Sjeff ibv_ack_cq_events(ctx->cq, num_cq_events); 801219820Sjeff 802219820Sjeff if (pp_close_ctx(ctx)) 803219820Sjeff return 1; 804219820Sjeff 805219820Sjeff ibv_free_device_list(dev_list); 806219820Sjeff free(rem_dest); 807219820Sjeff 808219820Sjeff return 0; 809219820Sjeff} 810