1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Topspin Communications. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#if HAVE_CONFIG_H 34219820Sjeff# include <config.h> 35219820Sjeff#endif /* HAVE_CONFIG_H */ 36219820Sjeff 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <stdlib.h> 39219820Sjeff#include <unistd.h> 40219820Sjeff#include <string.h> 41219820Sjeff#include <sys/types.h> 42219820Sjeff#include <sys/socket.h> 43219820Sjeff#include <sys/time.h> 44219820Sjeff#include <netdb.h> 45219820Sjeff#include <getopt.h> 46219820Sjeff#include <arpa/inet.h> 47219820Sjeff#include <time.h> 48219820Sjeff 49219820Sjeff#include "pingpong.h" 50219820Sjeff 51219820Sjeffenum { 52219820Sjeff PINGPONG_RECV_WRID = 1, 53219820Sjeff PINGPONG_SEND_WRID = 2, 54219820Sjeff}; 55219820Sjeff 56219820Sjeffstatic int page_size; 57219820Sjeff 58219820Sjeffstruct pingpong_context { 59219820Sjeff struct ibv_context *context; 60219820Sjeff struct ibv_comp_channel *channel; 61219820Sjeff struct ibv_pd *pd; 62219820Sjeff struct ibv_mr *mr; 63219820Sjeff struct ibv_cq *cq; 64219820Sjeff struct ibv_qp *qp; 65219820Sjeff struct ibv_ah *ah; 66219820Sjeff void *buf; 67219820Sjeff int size; 68219820Sjeff int rx_depth; 69219820Sjeff int pending; 70219820Sjeff struct ibv_port_attr portinfo; 71219820Sjeff}; 72219820Sjeff 73219820Sjeffstruct pingpong_dest { 74219820Sjeff int lid; 75219820Sjeff int qpn; 76219820Sjeff int psn; 77219820Sjeff union ibv_gid gid; 78219820Sjeff}; 79219820Sjeff 80219820Sjeffstatic int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 81219820Sjeff int sl, struct pingpong_dest *dest, int sgid_idx) 82219820Sjeff{ 83219820Sjeff struct ibv_ah_attr ah_attr = { 84219820Sjeff .is_global = 0, 85219820Sjeff .dlid = dest->lid, 86219820Sjeff .sl = sl, 87219820Sjeff .src_path_bits = 0, 88219820Sjeff .port_num = port 89219820Sjeff }; 90219820Sjeff struct ibv_qp_attr attr = { 91219820Sjeff .qp_state = IBV_QPS_RTR 92219820Sjeff }; 93219820Sjeff 94219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) { 95219820Sjeff fprintf(stderr, "Failed to modify QP to RTR\n"); 96219820Sjeff return 1; 97219820Sjeff } 98219820Sjeff 99219820Sjeff attr.qp_state = IBV_QPS_RTS; 100219820Sjeff attr.sq_psn = my_psn; 101219820Sjeff 102219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 103219820Sjeff IBV_QP_STATE | 104219820Sjeff IBV_QP_SQ_PSN)) { 105219820Sjeff fprintf(stderr, "Failed to modify QP to RTS\n"); 106219820Sjeff return 1; 107219820Sjeff } 108219820Sjeff 109219820Sjeff if (dest->gid.global.interface_id) { 110219820Sjeff ah_attr.is_global = 1; 111219820Sjeff ah_attr.grh.hop_limit = 1; 112219820Sjeff ah_attr.grh.dgid = dest->gid; 113219820Sjeff ah_attr.grh.sgid_index = sgid_idx; 114219820Sjeff } 115219820Sjeff 116219820Sjeff ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); 117219820Sjeff if (!ctx->ah) { 118219820Sjeff fprintf(stderr, "Failed to create AH\n"); 119219820Sjeff return 1; 120219820Sjeff } 121219820Sjeff 122219820Sjeff return 0; 123219820Sjeff} 124219820Sjeff 125219820Sjeffstatic struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, 126219820Sjeff const struct pingpong_dest *my_dest) 127219820Sjeff{ 128219820Sjeff struct addrinfo *res, *t; 129219820Sjeff struct addrinfo hints = { 130219820Sjeff .ai_family = AF_INET, 131219820Sjeff .ai_socktype = SOCK_STREAM 132219820Sjeff }; 133219820Sjeff char *service; 134219820Sjeff char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 135219820Sjeff int n; 136219820Sjeff int sockfd = -1; 137219820Sjeff struct pingpong_dest *rem_dest = NULL; 138219820Sjeff char gid[33]; 139219820Sjeff 140219820Sjeff if (asprintf(&service, "%d", port) < 0) 141219820Sjeff return NULL; 142219820Sjeff 143219820Sjeff n = getaddrinfo(servername, service, &hints, &res); 144219820Sjeff 145219820Sjeff if (n < 0) { 146219820Sjeff fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); 147219820Sjeff free(service); 148219820Sjeff return NULL; 149219820Sjeff } 150219820Sjeff 151219820Sjeff for (t = res; t; t = t->ai_next) { 152219820Sjeff sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 153219820Sjeff if (sockfd >= 0) { 154219820Sjeff if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) 155219820Sjeff break; 156219820Sjeff close(sockfd); 157219820Sjeff sockfd = -1; 158219820Sjeff } 159219820Sjeff } 160219820Sjeff 161219820Sjeff freeaddrinfo(res); 162219820Sjeff free(service); 163219820Sjeff 164219820Sjeff if (sockfd < 0) { 165219820Sjeff fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); 166219820Sjeff return NULL; 167219820Sjeff } 168219820Sjeff 169219820Sjeff gid_to_wire_gid(&my_dest->gid, gid); 170219820Sjeff sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 171219820Sjeff if (write(sockfd, msg, sizeof msg) != sizeof msg) { 172219820Sjeff fprintf(stderr, "Couldn't send local address\n"); 173219820Sjeff goto out; 174219820Sjeff } 175219820Sjeff 176219820Sjeff if (read(sockfd, msg, sizeof msg) != sizeof msg) { 177219820Sjeff perror("client read"); 178219820Sjeff fprintf(stderr, "Couldn't read remote address\n"); 179219820Sjeff goto out; 180219820Sjeff } 181219820Sjeff 182219820Sjeff write(sockfd, "done", sizeof "done"); 183219820Sjeff 184219820Sjeff rem_dest = malloc(sizeof *rem_dest); 185219820Sjeff if (!rem_dest) 186219820Sjeff goto out; 187219820Sjeff 188219820Sjeff sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 189219820Sjeff wire_gid_to_gid(gid, &rem_dest->gid); 190219820Sjeff 191219820Sjeffout: 192219820Sjeff close(sockfd); 193219820Sjeff return rem_dest; 194219820Sjeff} 195219820Sjeff 196219820Sjeffstatic struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 197219820Sjeff int ib_port, int port, int sl, 198219820Sjeff const struct pingpong_dest *my_dest, 199219820Sjeff int sgid_idx) 200219820Sjeff{ 201219820Sjeff struct addrinfo *res, *t; 202219820Sjeff struct addrinfo hints = { 203219820Sjeff .ai_flags = AI_PASSIVE, 204219820Sjeff .ai_family = AF_INET, 205219820Sjeff .ai_socktype = SOCK_STREAM 206219820Sjeff }; 207219820Sjeff char *service; 208219820Sjeff char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 209219820Sjeff int n; 210219820Sjeff int sockfd = -1, connfd; 211219820Sjeff struct pingpong_dest *rem_dest = NULL; 212219820Sjeff char gid[33]; 213219820Sjeff 214219820Sjeff if (asprintf(&service, "%d", port) < 0) 215219820Sjeff return NULL; 216219820Sjeff 217219820Sjeff n = getaddrinfo(NULL, service, &hints, &res); 218219820Sjeff 219219820Sjeff if (n < 0) { 220219820Sjeff fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); 221219820Sjeff free(service); 222219820Sjeff return NULL; 223219820Sjeff } 224219820Sjeff 225219820Sjeff for (t = res; t; t = t->ai_next) { 226219820Sjeff sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 227219820Sjeff if (sockfd >= 0) { 228219820Sjeff n = 1; 229219820Sjeff 230219820Sjeff setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); 231219820Sjeff 232219820Sjeff if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) 233219820Sjeff break; 234219820Sjeff close(sockfd); 235219820Sjeff sockfd = -1; 236219820Sjeff } 237219820Sjeff } 238219820Sjeff 239219820Sjeff freeaddrinfo(res); 240219820Sjeff free(service); 241219820Sjeff 242219820Sjeff if (sockfd < 0) { 243219820Sjeff fprintf(stderr, "Couldn't listen to port %d\n", port); 244219820Sjeff return NULL; 245219820Sjeff } 246219820Sjeff 247219820Sjeff listen(sockfd, 1); 248219820Sjeff connfd = accept(sockfd, NULL, 0); 249219820Sjeff close(sockfd); 250219820Sjeff if (connfd < 0) { 251219820Sjeff fprintf(stderr, "accept() failed\n"); 252219820Sjeff return NULL; 253219820Sjeff } 254219820Sjeff 255219820Sjeff n = read(connfd, msg, sizeof msg); 256219820Sjeff if (n != sizeof msg) { 257219820Sjeff perror("server read"); 258219820Sjeff fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); 259219820Sjeff goto out; 260219820Sjeff } 261219820Sjeff 262219820Sjeff rem_dest = malloc(sizeof *rem_dest); 263219820Sjeff if (!rem_dest) 264219820Sjeff goto out; 265219820Sjeff 266219820Sjeff sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 267219820Sjeff wire_gid_to_gid(gid, &rem_dest->gid); 268219820Sjeff 269219820Sjeff if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) { 270219820Sjeff fprintf(stderr, "Couldn't connect to remote QP\n"); 271219820Sjeff free(rem_dest); 272219820Sjeff rem_dest = NULL; 273219820Sjeff goto out; 274219820Sjeff } 275219820Sjeff 276219820Sjeff gid_to_wire_gid(&my_dest->gid, gid); 277219820Sjeff sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 278219820Sjeff if (write(connfd, msg, sizeof msg) != sizeof msg) { 279219820Sjeff fprintf(stderr, "Couldn't send local address\n"); 280219820Sjeff free(rem_dest); 281219820Sjeff rem_dest = NULL; 282219820Sjeff goto out; 283219820Sjeff } 284219820Sjeff 285219820Sjeff read(connfd, msg, sizeof msg); 286219820Sjeff 287219820Sjeffout: 288219820Sjeff close(connfd); 289219820Sjeff return rem_dest; 290219820Sjeff} 291219820Sjeff 292219820Sjeffstatic struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 293219820Sjeff int rx_depth, int port, 294219820Sjeff int use_event) 295219820Sjeff{ 296219820Sjeff struct pingpong_context *ctx; 297219820Sjeff 298219820Sjeff ctx = malloc(sizeof *ctx); 299219820Sjeff if (!ctx) 300219820Sjeff return NULL; 301219820Sjeff 302219820Sjeff ctx->size = size; 303219820Sjeff ctx->rx_depth = rx_depth; 304219820Sjeff 305219820Sjeff ctx->buf = malloc(roundup(size + 40, page_size)); 306219820Sjeff if (!ctx->buf) { 307219820Sjeff fprintf(stderr, "Couldn't allocate work buf.\n"); 308219820Sjeff return NULL; 309219820Sjeff } 310219820Sjeff 311219820Sjeff memset(ctx->buf, 0, size + 40); 312219820Sjeff 313219820Sjeff ctx->context = ibv_open_device(ib_dev); 314219820Sjeff if (!ctx->context) { 315219820Sjeff fprintf(stderr, "Couldn't get context for %s\n", 316219820Sjeff ibv_get_device_name(ib_dev)); 317219820Sjeff return NULL; 318219820Sjeff } 319219820Sjeff 320219820Sjeff if (use_event) { 321219820Sjeff ctx->channel = ibv_create_comp_channel(ctx->context); 322219820Sjeff if (!ctx->channel) { 323219820Sjeff fprintf(stderr, "Couldn't create completion channel\n"); 324219820Sjeff return NULL; 325219820Sjeff } 326219820Sjeff } else 327219820Sjeff ctx->channel = NULL; 328219820Sjeff 329219820Sjeff ctx->pd = ibv_alloc_pd(ctx->context); 330219820Sjeff if (!ctx->pd) { 331219820Sjeff fprintf(stderr, "Couldn't allocate PD\n"); 332219820Sjeff return NULL; 333219820Sjeff } 334219820Sjeff 335219820Sjeff ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE); 336219820Sjeff if (!ctx->mr) { 337219820Sjeff fprintf(stderr, "Couldn't register MR\n"); 338219820Sjeff return NULL; 339219820Sjeff } 340219820Sjeff 341219820Sjeff ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, 342219820Sjeff ctx->channel, 0); 343219820Sjeff if (!ctx->cq) { 344219820Sjeff fprintf(stderr, "Couldn't create CQ\n"); 345219820Sjeff return NULL; 346219820Sjeff } 347219820Sjeff 348219820Sjeff { 349219820Sjeff struct ibv_qp_init_attr attr = { 350219820Sjeff .send_cq = ctx->cq, 351219820Sjeff .recv_cq = ctx->cq, 352219820Sjeff .cap = { 353219820Sjeff .max_send_wr = 1, 354219820Sjeff .max_recv_wr = rx_depth, 355219820Sjeff .max_send_sge = 1, 356219820Sjeff .max_recv_sge = 1 357219820Sjeff }, 358219820Sjeff .qp_type = IBV_QPT_UD, 359219820Sjeff }; 360219820Sjeff 361219820Sjeff ctx->qp = ibv_create_qp(ctx->pd, &attr); 362219820Sjeff if (!ctx->qp) { 363219820Sjeff fprintf(stderr, "Couldn't create QP\n"); 364219820Sjeff return NULL; 365219820Sjeff } 366219820Sjeff } 367219820Sjeff 368219820Sjeff { 369219820Sjeff struct ibv_qp_attr attr = { 370219820Sjeff .qp_state = IBV_QPS_INIT, 371219820Sjeff .pkey_index = 0, 372219820Sjeff .port_num = port, 373219820Sjeff .qkey = 0x11111111 374219820Sjeff }; 375219820Sjeff 376219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 377219820Sjeff IBV_QP_STATE | 378219820Sjeff IBV_QP_PKEY_INDEX | 379219820Sjeff IBV_QP_PORT | 380219820Sjeff IBV_QP_QKEY)) { 381219820Sjeff fprintf(stderr, "Failed to modify QP to INIT\n"); 382219820Sjeff return NULL; 383219820Sjeff } 384219820Sjeff } 385219820Sjeff 386219820Sjeff return ctx; 387219820Sjeff} 388219820Sjeff 389219820Sjeffint pp_close_ctx(struct pingpong_context *ctx) 390219820Sjeff{ 391219820Sjeff if (ibv_destroy_qp(ctx->qp)) { 392219820Sjeff fprintf(stderr, "Couldn't destroy QP\n"); 393219820Sjeff return 1; 394219820Sjeff } 395219820Sjeff 396219820Sjeff if (ibv_destroy_cq(ctx->cq)) { 397219820Sjeff fprintf(stderr, "Couldn't destroy CQ\n"); 398219820Sjeff return 1; 399219820Sjeff } 400219820Sjeff 401219820Sjeff if (ibv_dereg_mr(ctx->mr)) { 402219820Sjeff fprintf(stderr, "Couldn't deregister MR\n"); 403219820Sjeff return 1; 404219820Sjeff } 405219820Sjeff 406219820Sjeff if (ibv_destroy_ah(ctx->ah)) { 407219820Sjeff fprintf(stderr, "Couldn't destroy AH\n"); 408219820Sjeff return 1; 409219820Sjeff } 410219820Sjeff 411219820Sjeff if (ibv_dealloc_pd(ctx->pd)) { 412219820Sjeff fprintf(stderr, "Couldn't deallocate PD\n"); 413219820Sjeff return 1; 414219820Sjeff } 415219820Sjeff 416219820Sjeff if (ctx->channel) { 417219820Sjeff if (ibv_destroy_comp_channel(ctx->channel)) { 418219820Sjeff fprintf(stderr, "Couldn't destroy completion channel\n"); 419219820Sjeff return 1; 420219820Sjeff } 421219820Sjeff } 422219820Sjeff 423219820Sjeff if (ibv_close_device(ctx->context)) { 424219820Sjeff fprintf(stderr, "Couldn't release context\n"); 425219820Sjeff return 1; 426219820Sjeff } 427219820Sjeff 428219820Sjeff free(ctx->buf); 429219820Sjeff free(ctx); 430219820Sjeff 431219820Sjeff return 0; 432219820Sjeff} 433219820Sjeff 434219820Sjeffstatic int pp_post_recv(struct pingpong_context *ctx, int n) 435219820Sjeff{ 436219820Sjeff struct ibv_sge list = { 437219820Sjeff .addr = (uintptr_t) ctx->buf, 438219820Sjeff .length = ctx->size + 40, 439219820Sjeff .lkey = ctx->mr->lkey 440219820Sjeff }; 441219820Sjeff struct ibv_recv_wr wr = { 442219820Sjeff .wr_id = PINGPONG_RECV_WRID, 443219820Sjeff .sg_list = &list, 444219820Sjeff .num_sge = 1, 445219820Sjeff }; 446219820Sjeff struct ibv_recv_wr *bad_wr; 447219820Sjeff int i; 448219820Sjeff 449219820Sjeff for (i = 0; i < n; ++i) 450219820Sjeff if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 451219820Sjeff break; 452219820Sjeff 453219820Sjeff return i; 454219820Sjeff} 455219820Sjeff 456219820Sjeffstatic int pp_post_send(struct pingpong_context *ctx, uint32_t qpn) 457219820Sjeff{ 458219820Sjeff struct ibv_sge list = { 459219820Sjeff .addr = (uintptr_t) ctx->buf + 40, 460219820Sjeff .length = ctx->size, 461219820Sjeff .lkey = ctx->mr->lkey 462219820Sjeff }; 463219820Sjeff struct ibv_send_wr wr = { 464219820Sjeff .wr_id = PINGPONG_SEND_WRID, 465219820Sjeff .sg_list = &list, 466219820Sjeff .num_sge = 1, 467219820Sjeff .opcode = IBV_WR_SEND, 468219820Sjeff .send_flags = IBV_SEND_SIGNALED, 469219820Sjeff .wr = { 470219820Sjeff .ud = { 471219820Sjeff .ah = ctx->ah, 472219820Sjeff .remote_qpn = qpn, 473219820Sjeff .remote_qkey = 0x11111111 474219820Sjeff } 475219820Sjeff } 476219820Sjeff }; 477219820Sjeff struct ibv_send_wr *bad_wr; 478219820Sjeff 479219820Sjeff return ibv_post_send(ctx->qp, &wr, &bad_wr); 480219820Sjeff} 481219820Sjeff 482219820Sjeffstatic void usage(const char *argv0) 483219820Sjeff{ 484219820Sjeff printf("Usage:\n"); 485219820Sjeff printf(" %s start a server and wait for connection\n", argv0); 486219820Sjeff printf(" %s <host> connect to server at <host>\n", argv0); 487219820Sjeff printf("\n"); 488219820Sjeff printf("Options:\n"); 489219820Sjeff printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 490219820Sjeff printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 491219820Sjeff printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 492219820Sjeff printf(" -s, --size=<size> size of message to exchange (default 1024)\n"); 493219820Sjeff printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 494219820Sjeff printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 495219820Sjeff printf(" -e, --events sleep on CQ events (default poll)\n"); 496219820Sjeff printf(" -g, --gid-idx=<gid index> local port gid index\n"); 497219820Sjeff} 498219820Sjeff 499219820Sjeffint main(int argc, char *argv[]) 500219820Sjeff{ 501219820Sjeff struct ibv_device **dev_list; 502219820Sjeff struct ibv_device *ib_dev; 503219820Sjeff struct pingpong_context *ctx; 504219820Sjeff struct pingpong_dest my_dest; 505219820Sjeff struct pingpong_dest *rem_dest; 506219820Sjeff struct timeval start, end; 507219820Sjeff char *ib_devname = NULL; 508219820Sjeff char *servername = NULL; 509219820Sjeff int port = 18515; 510219820Sjeff int ib_port = 1; 511219820Sjeff int size = 1024; 512219820Sjeff int rx_depth = 500; 513219820Sjeff int iters = 1000; 514219820Sjeff int use_event = 0; 515219820Sjeff int routs; 516219820Sjeff int rcnt, scnt; 517219820Sjeff int num_cq_events = 0; 518219820Sjeff int sl = 0; 519219820Sjeff int gidx = -1; 520219820Sjeff char gid[33]; 521219820Sjeff 522219820Sjeff srand48(getpid() * time(NULL)); 523219820Sjeff 524219820Sjeff while (1) { 525219820Sjeff int c; 526219820Sjeff 527219820Sjeff static struct option long_options[] = { 528219820Sjeff { .name = "port", .has_arg = 1, .val = 'p' }, 529219820Sjeff { .name = "ib-dev", .has_arg = 1, .val = 'd' }, 530219820Sjeff { .name = "ib-port", .has_arg = 1, .val = 'i' }, 531219820Sjeff { .name = "size", .has_arg = 1, .val = 's' }, 532219820Sjeff { .name = "rx-depth", .has_arg = 1, .val = 'r' }, 533219820Sjeff { .name = "iters", .has_arg = 1, .val = 'n' }, 534219820Sjeff { .name = "sl", .has_arg = 1, .val = 'l' }, 535219820Sjeff { .name = "events", .has_arg = 0, .val = 'e' }, 536219820Sjeff { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 537219820Sjeff { 0 } 538219820Sjeff }; 539219820Sjeff 540219820Sjeff c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL); 541219820Sjeff if (c == -1) 542219820Sjeff break; 543219820Sjeff 544219820Sjeff switch (c) { 545219820Sjeff case 'p': 546219820Sjeff port = strtol(optarg, NULL, 0); 547219820Sjeff if (port < 0 || port > 65535) { 548219820Sjeff usage(argv[0]); 549219820Sjeff return 1; 550219820Sjeff } 551219820Sjeff break; 552219820Sjeff 553219820Sjeff case 'd': 554219820Sjeff ib_devname = strdup(optarg); 555219820Sjeff break; 556219820Sjeff 557219820Sjeff case 'i': 558219820Sjeff ib_port = strtol(optarg, NULL, 0); 559219820Sjeff if (ib_port < 0) { 560219820Sjeff usage(argv[0]); 561219820Sjeff return 1; 562219820Sjeff } 563219820Sjeff break; 564219820Sjeff 565219820Sjeff case 's': 566219820Sjeff size = strtol(optarg, NULL, 0); 567219820Sjeff break; 568219820Sjeff 569219820Sjeff case 'r': 570219820Sjeff rx_depth = strtol(optarg, NULL, 0); 571219820Sjeff break; 572219820Sjeff 573219820Sjeff case 'n': 574219820Sjeff iters = strtol(optarg, NULL, 0); 575219820Sjeff break; 576219820Sjeff 577219820Sjeff case 'l': 578219820Sjeff sl = strtol(optarg, NULL, 0); 579219820Sjeff break; 580219820Sjeff 581219820Sjeff case 'e': 582219820Sjeff ++use_event; 583219820Sjeff break; 584219820Sjeff 585219820Sjeff case 'g': 586219820Sjeff gidx = strtol(optarg, NULL, 0); 587219820Sjeff break; 588219820Sjeff 589219820Sjeff default: 590219820Sjeff usage(argv[0]); 591219820Sjeff return 1; 592219820Sjeff } 593219820Sjeff } 594219820Sjeff 595219820Sjeff if (optind == argc - 1) 596219820Sjeff servername = strdup(argv[optind]); 597219820Sjeff else if (optind < argc) { 598219820Sjeff usage(argv[0]); 599219820Sjeff return 1; 600219820Sjeff } 601219820Sjeff 602219820Sjeff page_size = sysconf(_SC_PAGESIZE); 603219820Sjeff 604219820Sjeff dev_list = ibv_get_device_list(NULL); 605219820Sjeff if (!dev_list) { 606219820Sjeff perror("Failed to get IB devices list"); 607219820Sjeff return 1; 608219820Sjeff } 609219820Sjeff 610219820Sjeff if (!ib_devname) { 611219820Sjeff ib_dev = *dev_list; 612219820Sjeff if (!ib_dev) { 613219820Sjeff fprintf(stderr, "No IB devices found\n"); 614219820Sjeff return 1; 615219820Sjeff } 616219820Sjeff } else { 617219820Sjeff int i; 618219820Sjeff for (i = 0; dev_list[i]; ++i) 619219820Sjeff if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) 620219820Sjeff break; 621219820Sjeff ib_dev = dev_list[i]; 622219820Sjeff if (!ib_dev) { 623219820Sjeff fprintf(stderr, "IB device %s not found\n", ib_devname); 624219820Sjeff return 1; 625219820Sjeff } 626219820Sjeff } 627219820Sjeff 628219820Sjeff ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 629219820Sjeff if (!ctx) 630219820Sjeff return 1; 631219820Sjeff 632219820Sjeff routs = pp_post_recv(ctx, ctx->rx_depth); 633219820Sjeff if (routs < ctx->rx_depth) { 634219820Sjeff fprintf(stderr, "Couldn't post receive (%d)\n", routs); 635219820Sjeff return 1; 636219820Sjeff } 637219820Sjeff 638219820Sjeff if (use_event) 639219820Sjeff if (ibv_req_notify_cq(ctx->cq, 0)) { 640219820Sjeff fprintf(stderr, "Couldn't request CQ notification\n"); 641219820Sjeff return 1; 642219820Sjeff } 643219820Sjeff 644219820Sjeff if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 645219820Sjeff fprintf(stderr, "Couldn't get port info\n"); 646219820Sjeff return 1; 647219820Sjeff } 648219820Sjeff my_dest.lid = ctx->portinfo.lid; 649219820Sjeff 650219820Sjeff my_dest.qpn = ctx->qp->qp_num; 651219820Sjeff my_dest.psn = lrand48() & 0xffffff; 652219820Sjeff 653219820Sjeff if (gidx >= 0) { 654219820Sjeff if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 655219820Sjeff fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 656219820Sjeff return 1; 657219820Sjeff } 658219820Sjeff } else 659219820Sjeff memset(&my_dest.gid, 0, sizeof my_dest.gid); 660219820Sjeff 661219820Sjeff inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 662219820Sjeff printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", 663219820Sjeff my_dest.lid, my_dest.qpn, my_dest.psn, gid); 664219820Sjeff 665219820Sjeff if (servername) 666219820Sjeff rem_dest = pp_client_exch_dest(servername, port, &my_dest); 667219820Sjeff else 668219820Sjeff rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx); 669219820Sjeff 670219820Sjeff if (!rem_dest) 671219820Sjeff return 1; 672219820Sjeff 673219820Sjeff inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 674219820Sjeff printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 675219820Sjeff rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 676219820Sjeff 677219820Sjeff if (servername) 678219820Sjeff if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) 679219820Sjeff return 1; 680219820Sjeff 681219820Sjeff ctx->pending = PINGPONG_RECV_WRID; 682219820Sjeff 683219820Sjeff if (servername) { 684219820Sjeff if (pp_post_send(ctx, rem_dest->qpn)) { 685219820Sjeff fprintf(stderr, "Couldn't post send\n"); 686219820Sjeff return 1; 687219820Sjeff } 688219820Sjeff ctx->pending |= PINGPONG_SEND_WRID; 689219820Sjeff } 690219820Sjeff 691219820Sjeff if (gettimeofday(&start, NULL)) { 692219820Sjeff perror("gettimeofday"); 693219820Sjeff return 1; 694219820Sjeff } 695219820Sjeff 696219820Sjeff rcnt = scnt = 0; 697219820Sjeff while (rcnt < iters || scnt < iters) { 698219820Sjeff if (use_event) { 699219820Sjeff struct ibv_cq *ev_cq; 700219820Sjeff void *ev_ctx; 701219820Sjeff 702219820Sjeff if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 703219820Sjeff fprintf(stderr, "Failed to get cq_event\n"); 704219820Sjeff return 1; 705219820Sjeff } 706219820Sjeff 707219820Sjeff ++num_cq_events; 708219820Sjeff 709219820Sjeff if (ev_cq != ctx->cq) { 710219820Sjeff fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); 711219820Sjeff return 1; 712219820Sjeff } 713219820Sjeff 714219820Sjeff if (ibv_req_notify_cq(ctx->cq, 0)) { 715219820Sjeff fprintf(stderr, "Couldn't request CQ notification\n"); 716219820Sjeff return 1; 717219820Sjeff } 718219820Sjeff } 719219820Sjeff 720219820Sjeff { 721219820Sjeff struct ibv_wc wc[2]; 722219820Sjeff int ne, i; 723219820Sjeff 724219820Sjeff do { 725219820Sjeff ne = ibv_poll_cq(ctx->cq, 2, wc); 726219820Sjeff if (ne < 0) { 727219820Sjeff fprintf(stderr, "poll CQ failed %d\n", ne); 728219820Sjeff return 1; 729219820Sjeff } 730219820Sjeff } while (!use_event && ne < 1); 731219820Sjeff 732219820Sjeff for (i = 0; i < ne; ++i) { 733219820Sjeff if (wc[i].status != IBV_WC_SUCCESS) { 734219820Sjeff fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", 735219820Sjeff ibv_wc_status_str(wc[i].status), 736219820Sjeff wc[i].status, (int) wc[i].wr_id); 737219820Sjeff return 1; 738219820Sjeff } 739219820Sjeff 740219820Sjeff switch ((int) wc[i].wr_id) { 741219820Sjeff case PINGPONG_SEND_WRID: 742219820Sjeff ++scnt; 743219820Sjeff break; 744219820Sjeff 745219820Sjeff case PINGPONG_RECV_WRID: 746219820Sjeff if (--routs <= 1) { 747219820Sjeff routs += pp_post_recv(ctx, ctx->rx_depth - routs); 748219820Sjeff if (routs < ctx->rx_depth) { 749219820Sjeff fprintf(stderr, 750219820Sjeff "Couldn't post receive (%d)\n", 751219820Sjeff routs); 752219820Sjeff return 1; 753219820Sjeff } 754219820Sjeff } 755219820Sjeff 756219820Sjeff ++rcnt; 757219820Sjeff break; 758219820Sjeff 759219820Sjeff default: 760219820Sjeff fprintf(stderr, "Completion for unknown wr_id %d\n", 761219820Sjeff (int) wc[i].wr_id); 762219820Sjeff return 1; 763219820Sjeff } 764219820Sjeff 765219820Sjeff ctx->pending &= ~(int) wc[i].wr_id; 766219820Sjeff if (scnt < iters && !ctx->pending) { 767219820Sjeff if (pp_post_send(ctx, rem_dest->qpn)) { 768219820Sjeff fprintf(stderr, "Couldn't post send\n"); 769219820Sjeff return 1; 770219820Sjeff } 771219820Sjeff ctx->pending = PINGPONG_RECV_WRID | 772219820Sjeff PINGPONG_SEND_WRID; 773219820Sjeff } 774219820Sjeff } 775219820Sjeff } 776219820Sjeff } 777219820Sjeff 778219820Sjeff if (gettimeofday(&end, NULL)) { 779219820Sjeff perror("gettimeofday"); 780219820Sjeff return 1; 781219820Sjeff } 782219820Sjeff 783219820Sjeff { 784219820Sjeff float usec = (end.tv_sec - start.tv_sec) * 1000000 + 785219820Sjeff (end.tv_usec - start.tv_usec); 786219820Sjeff long long bytes = (long long) size * iters * 2; 787219820Sjeff 788219820Sjeff printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", 789219820Sjeff bytes, usec / 1000000., bytes * 8. / usec); 790219820Sjeff printf("%d iters in %.2f seconds = %.2f usec/iter\n", 791219820Sjeff iters, usec / 1000000., usec / iters); 792219820Sjeff } 793219820Sjeff 794219820Sjeff ibv_ack_cq_events(ctx->cq, num_cq_events); 795219820Sjeff 796219820Sjeff if (pp_close_ctx(ctx)) 797219820Sjeff return 1; 798219820Sjeff 799219820Sjeff ibv_free_device_list(dev_list); 800219820Sjeff free(rem_dest); 801219820Sjeff 802219820Sjeff return 0; 803219820Sjeff} 804