uc_pingpong.c revision 219820
1178825Sdfr/* 2233294Sstas * Copyright (c) 2005 Topspin Communications. All rights reserved. 3233294Sstas * 4233294Sstas * This software is available to you under a choice of one of two 5178825Sdfr * licenses. You may choose to be licensed under the terms of the GNU 6233294Sstas * General Public License (GPL) Version 2, available from the file 7233294Sstas * COPYING in the main directory of this source tree, or the 8233294Sstas * OpenIB.org BSD license below: 9178825Sdfr * 10233294Sstas * Redistribution and use in source and binary forms, with or 11233294Sstas * without modification, are permitted provided that the following 12178825Sdfr * conditions are met: 13233294Sstas * 14233294Sstas * - Redistributions of source code must retain the above 15233294Sstas * copyright notice, this list of conditions and the following 16178825Sdfr * disclaimer. 17233294Sstas * 18233294Sstas * - Redistributions in binary form must reproduce the above 19233294Sstas * copyright notice, this list of conditions and the following 20178825Sdfr * disclaimer in the documentation and/or other materials 21233294Sstas * provided with the distribution. 22233294Sstas * 23233294Sstas * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24233294Sstas * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25233294Sstas * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26233294Sstas * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27233294Sstas * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28233294Sstas * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29233294Sstas * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30233294Sstas * SOFTWARE. 31233294Sstas */ 32178825Sdfr 33178825Sdfr#if HAVE_CONFIG_H 34178825Sdfr# include <config.h> 35178825Sdfr#endif /* HAVE_CONFIG_H */ 36178825Sdfr 37178825Sdfr#include <stdio.h> 38178825Sdfr#include <stdlib.h> 39178825Sdfr#include <unistd.h> 40178825Sdfr#include <string.h> 41178825Sdfr#include <sys/types.h> 42178825Sdfr#include <sys/socket.h> 43178825Sdfr#include <sys/time.h> 44178825Sdfr#include <netdb.h> 45178825Sdfr#include <getopt.h> 46178825Sdfr#include <arpa/inet.h> 47178825Sdfr#include <time.h> 48178825Sdfr 49178825Sdfr#include "pingpong.h" 50178825Sdfr 51178825Sdfrenum { 52178825Sdfr PINGPONG_RECV_WRID = 1, 53178825Sdfr PINGPONG_SEND_WRID = 2, 54178825Sdfr}; 55178825Sdfr 56178825Sdfrstatic int page_size; 57178825Sdfr 58178825Sdfrstruct pingpong_context { 59178825Sdfr struct ibv_context *context; 60178825Sdfr struct ibv_comp_channel *channel; 61178825Sdfr struct ibv_pd *pd; 62178825Sdfr struct ibv_mr *mr; 63178825Sdfr struct ibv_cq *cq; 64233294Sstas struct ibv_qp *qp; 65178825Sdfr void *buf; 66178825Sdfr int size; 67178825Sdfr int rx_depth; 68178825Sdfr int pending; 69178825Sdfr struct ibv_port_attr portinfo; 70178825Sdfr}; 71178825Sdfr 72178825Sdfrstruct pingpong_dest { 73178825Sdfr int lid; 74178825Sdfr int qpn; 75178825Sdfr int psn; 76178825Sdfr union ibv_gid gid; 77178825Sdfr}; 78178825Sdfr 79178825Sdfrstatic int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 80178825Sdfr enum ibv_mtu mtu, int sl, 81178825Sdfr struct pingpong_dest *dest, int sgid_idx) 82178825Sdfr{ 83178825Sdfr struct ibv_qp_attr attr = { 84178825Sdfr .qp_state = IBV_QPS_RTR, 85178825Sdfr .path_mtu = mtu, 86178825Sdfr .dest_qp_num = dest->qpn, 87178825Sdfr .rq_psn = dest->psn, 88178825Sdfr .ah_attr = { 89178825Sdfr .is_global = 0, 90178825Sdfr .dlid = dest->lid, 91178825Sdfr .sl = sl, 92178825Sdfr .src_path_bits = 0, 93178825Sdfr .port_num = port 94178825Sdfr } 95178825Sdfr }; 96178825Sdfr 97178825Sdfr if (dest->gid.global.interface_id) { 98178825Sdfr attr.ah_attr.is_global = 1; 99178825Sdfr attr.ah_attr.grh.hop_limit = 1; 100178825Sdfr attr.ah_attr.grh.dgid = dest->gid; 101178825Sdfr attr.ah_attr.grh.sgid_index = sgid_idx; 102178825Sdfr } 103178825Sdfr 104178825Sdfr if (ibv_modify_qp(ctx->qp, &attr, 105233294Sstas IBV_QP_STATE | 106233294Sstas IBV_QP_AV | 107233294Sstas IBV_QP_PATH_MTU | 108233294Sstas IBV_QP_DEST_QPN | 109178825Sdfr IBV_QP_RQ_PSN)) { 110178825Sdfr fprintf(stderr, "Failed to modify QP to RTR\n"); 111178825Sdfr return 1; 112178825Sdfr } 113178825Sdfr 114178825Sdfr attr.qp_state = IBV_QPS_RTS; 115233294Sstas attr.sq_psn = my_psn; 116178825Sdfr if (ibv_modify_qp(ctx->qp, &attr, 117178825Sdfr IBV_QP_STATE | 118178825Sdfr IBV_QP_SQ_PSN)) { 119178825Sdfr fprintf(stderr, "Failed to modify QP to RTS\n"); 120178825Sdfr return 1; 121178825Sdfr } 122233294Sstas 123178825Sdfr return 0; 124178825Sdfr} 125178825Sdfr 126178825Sdfrstatic struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, 127178825Sdfr const struct pingpong_dest *my_dest) 128178825Sdfr{ 129178825Sdfr struct addrinfo *res, *t; 130178825Sdfr struct addrinfo hints = { 131178825Sdfr .ai_family = AF_INET, 132178825Sdfr .ai_socktype = SOCK_STREAM 133178825Sdfr }; 134178825Sdfr char *service; 135178825Sdfr char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 136178825Sdfr int n; 137178825Sdfr int sockfd = -1; 138178825Sdfr struct pingpong_dest *rem_dest = NULL; 139178825Sdfr char gid[33]; 140178825Sdfr 141178825Sdfr if (asprintf(&service, "%d", port) < 0) 142178825Sdfr return NULL; 143178825Sdfr 144178825Sdfr n = getaddrinfo(servername, service, &hints, &res); 145178825Sdfr 146178825Sdfr if (n < 0) { 147178825Sdfr fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); 148178825Sdfr free(service); 149178825Sdfr return NULL; 150178825Sdfr } 151178825Sdfr 152178825Sdfr for (t = res; t; t = t->ai_next) { 153178825Sdfr sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 154178825Sdfr if (sockfd >= 0) { 155178825Sdfr if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) 156178825Sdfr break; 157178825Sdfr close(sockfd); 158178825Sdfr sockfd = -1; 159178825Sdfr } 160233294Sstas } 161178825Sdfr 162178825Sdfr freeaddrinfo(res); 163178825Sdfr free(service); 164178825Sdfr 165178825Sdfr if (sockfd < 0) { 166178825Sdfr fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); 167178825Sdfr return NULL; 168178825Sdfr } 169178825Sdfr 170178825Sdfr gid_to_wire_gid(&my_dest->gid, gid); 171178825Sdfr sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 172178825Sdfr if (write(sockfd, msg, sizeof msg) != sizeof msg) { 173178825Sdfr fprintf(stderr, "Couldn't send local address\n"); 174178825Sdfr goto out; 175178825Sdfr } 176178825Sdfr 177178825Sdfr if (read(sockfd, msg, sizeof msg) != sizeof msg) { 178178825Sdfr perror("client read"); 179178825Sdfr fprintf(stderr, "Couldn't read remote address\n"); 180178825Sdfr goto out; 181178825Sdfr } 182178825Sdfr 183178825Sdfr write(sockfd, "done", sizeof "done"); 184178825Sdfr 185178825Sdfr rem_dest = malloc(sizeof *rem_dest); 186178825Sdfr if (!rem_dest) 187178825Sdfr goto out; 188178825Sdfr 189178825Sdfr sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 190178825Sdfr wire_gid_to_gid(gid, &rem_dest->gid); 191178825Sdfr 192178825Sdfrout: 193178825Sdfr close(sockfd); 194178825Sdfr return rem_dest; 195178825Sdfr} 196178825Sdfr 197178825Sdfrstatic struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 198178825Sdfr int ib_port, enum ibv_mtu mtu, 199178825Sdfr int port, int sl, 200178825Sdfr const struct pingpong_dest *my_dest, 201178825Sdfr int sgid_idx) 202178825Sdfr{ 203178825Sdfr struct addrinfo *res, *t; 204178825Sdfr struct addrinfo hints = { 205178825Sdfr .ai_flags = AI_PASSIVE, 206178825Sdfr .ai_family = AF_INET, 207178825Sdfr .ai_socktype = SOCK_STREAM 208178825Sdfr }; 209178825Sdfr char *service; 210178825Sdfr char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 211178825Sdfr int n; 212178825Sdfr int sockfd = -1, connfd; 213178825Sdfr struct pingpong_dest *rem_dest = NULL; 214178825Sdfr char gid[33]; 215178825Sdfr 216178825Sdfr if (asprintf(&service, "%d", port) < 0) 217178825Sdfr return NULL; 218178825Sdfr 219178825Sdfr n = getaddrinfo(NULL, service, &hints, &res); 220178825Sdfr 221178825Sdfr if (n < 0) { 222178825Sdfr fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); 223178825Sdfr free(service); 224178825Sdfr return NULL; 225178825Sdfr } 226178825Sdfr 227178825Sdfr for (t = res; t; t = t->ai_next) { 228178825Sdfr sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 229233294Sstas if (sockfd >= 0) { 230178825Sdfr n = 1; 231178825Sdfr 232233294Sstas setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); 233233294Sstas 234233294Sstas if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) 235178825Sdfr break; 236178825Sdfr close(sockfd); 237178825Sdfr sockfd = -1; 238178825Sdfr } 239178825Sdfr } 240178825Sdfr 241178825Sdfr freeaddrinfo(res); 242178825Sdfr free(service); 243178825Sdfr 244178825Sdfr if (sockfd < 0) { 245178825Sdfr fprintf(stderr, "Couldn't listen to port %d\n", port); 246178825Sdfr return NULL; 247178825Sdfr } 248178825Sdfr 249178825Sdfr listen(sockfd, 1); 250178825Sdfr connfd = accept(sockfd, NULL, 0); 251178825Sdfr close(sockfd); 252178825Sdfr if (connfd < 0) { 253178825Sdfr fprintf(stderr, "accept() failed\n"); 254178825Sdfr return NULL; 255178825Sdfr } 256178825Sdfr 257178825Sdfr n = read(connfd, msg, sizeof msg); 258178825Sdfr if (n != sizeof msg) { 259178825Sdfr perror("server read"); 260178825Sdfr fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); 261178825Sdfr goto out; 262178825Sdfr } 263178825Sdfr 264178825Sdfr rem_dest = malloc(sizeof *rem_dest); 265178825Sdfr if (!rem_dest) 266178825Sdfr goto out; 267 268 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 269 wire_gid_to_gid(gid, &rem_dest->gid); 270 271 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 272 fprintf(stderr, "Couldn't connect to remote QP\n"); 273 free(rem_dest); 274 rem_dest = NULL; 275 goto out; 276 } 277 278 gid_to_wire_gid(&my_dest->gid, gid); 279 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 280 if (write(connfd, msg, sizeof msg) != sizeof msg) { 281 fprintf(stderr, "Couldn't send local address\n"); 282 free(rem_dest); 283 rem_dest = NULL; 284 goto out; 285 } 286 287 read(connfd, msg, sizeof msg); 288 289out: 290 close(connfd); 291 return rem_dest; 292} 293 294static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 295 int rx_depth, int port, 296 int use_event) 297{ 298 struct pingpong_context *ctx; 299 300 ctx = calloc(1, sizeof *ctx); 301 if (!ctx) 302 return NULL; 303 304 ctx->size = size; 305 ctx->rx_depth = rx_depth; 306 307 ctx->buf = malloc(roundup(size, page_size)); 308 if (!ctx->buf) { 309 fprintf(stderr, "Couldn't allocate work buf.\n"); 310 return NULL; 311 } 312 313 memset(ctx->buf, 0, size); 314 315 ctx->context = ibv_open_device(ib_dev); 316 if (!ctx->context) { 317 fprintf(stderr, "Couldn't get context for %s\n", 318 ibv_get_device_name(ib_dev)); 319 return NULL; 320 } 321 322 if (use_event) { 323 ctx->channel = ibv_create_comp_channel(ctx->context); 324 if (!ctx->channel) { 325 fprintf(stderr, "Couldn't create completion channel\n"); 326 return NULL; 327 } 328 } else 329 ctx->channel = NULL; 330 331 ctx->pd = ibv_alloc_pd(ctx->context); 332 if (!ctx->pd) { 333 fprintf(stderr, "Couldn't allocate PD\n"); 334 return NULL; 335 } 336 337 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); 338 if (!ctx->mr) { 339 fprintf(stderr, "Couldn't register MR\n"); 340 return NULL; 341 } 342 343 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, 344 ctx->channel, 0); 345 if (!ctx->cq) { 346 fprintf(stderr, "Couldn't create CQ\n"); 347 return NULL; 348 } 349 350 { 351 struct ibv_qp_init_attr attr = { 352 .send_cq = ctx->cq, 353 .recv_cq = ctx->cq, 354 .cap = { 355 .max_send_wr = 1, 356 .max_recv_wr = rx_depth, 357 .max_send_sge = 1, 358 .max_recv_sge = 1 359 }, 360 .qp_type = IBV_QPT_UC 361 }; 362 363 ctx->qp = ibv_create_qp(ctx->pd, &attr); 364 if (!ctx->qp) { 365 fprintf(stderr, "Couldn't create QP\n"); 366 return NULL; 367 } 368 } 369 370 { 371 struct ibv_qp_attr attr = { 372 .qp_state = IBV_QPS_INIT, 373 .pkey_index = 0, 374 .port_num = port, 375 .qp_access_flags = 0 376 }; 377 378 if (ibv_modify_qp(ctx->qp, &attr, 379 IBV_QP_STATE | 380 IBV_QP_PKEY_INDEX | 381 IBV_QP_PORT | 382 IBV_QP_ACCESS_FLAGS)) { 383 fprintf(stderr, "Failed to modify QP to INIT\n"); 384 return NULL; 385 } 386 } 387 388 return ctx; 389} 390 391int pp_close_ctx(struct pingpong_context *ctx) 392{ 393 if (ibv_destroy_qp(ctx->qp)) { 394 fprintf(stderr, "Couldn't destroy QP\n"); 395 return 1; 396 } 397 398 if (ibv_destroy_cq(ctx->cq)) { 399 fprintf(stderr, "Couldn't destroy CQ\n"); 400 return 1; 401 } 402 403 if (ibv_dereg_mr(ctx->mr)) { 404 fprintf(stderr, "Couldn't deregister MR\n"); 405 return 1; 406 } 407 408 if (ibv_dealloc_pd(ctx->pd)) { 409 fprintf(stderr, "Couldn't deallocate PD\n"); 410 return 1; 411 } 412 413 if (ctx->channel) { 414 if (ibv_destroy_comp_channel(ctx->channel)) { 415 fprintf(stderr, "Couldn't destroy completion channel\n"); 416 return 1; 417 } 418 } 419 420 if (ibv_close_device(ctx->context)) { 421 fprintf(stderr, "Couldn't release context\n"); 422 return 1; 423 } 424 425 free(ctx->buf); 426 free(ctx); 427 428 return 0; 429} 430 431static int pp_post_recv(struct pingpong_context *ctx, int n) 432{ 433 struct ibv_sge list = { 434 .addr = (uintptr_t) ctx->buf, 435 .length = ctx->size, 436 .lkey = ctx->mr->lkey 437 }; 438 struct ibv_recv_wr wr = { 439 .wr_id = PINGPONG_RECV_WRID, 440 .sg_list = &list, 441 .num_sge = 1, 442 }; 443 struct ibv_recv_wr *bad_wr; 444 int i; 445 446 for (i = 0; i < n; ++i) 447 if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 448 break; 449 450 return i; 451} 452 453static int pp_post_send(struct pingpong_context *ctx) 454{ 455 struct ibv_sge list = { 456 .addr = (uintptr_t) ctx->buf, 457 .length = ctx->size, 458 .lkey = ctx->mr->lkey 459 }; 460 struct ibv_send_wr wr = { 461 .wr_id = PINGPONG_SEND_WRID, 462 .sg_list = &list, 463 .num_sge = 1, 464 .opcode = IBV_WR_SEND, 465 .send_flags = IBV_SEND_SIGNALED, 466 }; 467 struct ibv_send_wr *bad_wr; 468 469 return ibv_post_send(ctx->qp, &wr, &bad_wr); 470} 471 472static void usage(const char *argv0) 473{ 474 printf("Usage:\n"); 475 printf(" %s start a server and wait for connection\n", argv0); 476 printf(" %s <host> connect to server at <host>\n", argv0); 477 printf("\n"); 478 printf("Options:\n"); 479 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 480 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 481 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 482 printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); 483 printf(" -m, --mtu=<size> path MTU (default 1024)\n"); 484 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 485 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 486 printf(" -l, --sl=<sl> service level value\n"); 487 printf(" -e, --events sleep on CQ events (default poll)\n"); 488 printf(" -g, --gid-idx=<gid index> local port gid index\n"); 489} 490 491int main(int argc, char *argv[]) 492{ 493 struct ibv_device **dev_list; 494 struct ibv_device *ib_dev; 495 struct pingpong_context *ctx; 496 struct pingpong_dest my_dest; 497 struct pingpong_dest *rem_dest; 498 struct timeval start, end; 499 char *ib_devname = NULL; 500 char *servername = NULL; 501 int port = 18515; 502 int ib_port = 1; 503 int size = 4096; 504 enum ibv_mtu mtu = IBV_MTU_1024; 505 int rx_depth = 500; 506 int iters = 1000; 507 int use_event = 0; 508 int routs; 509 int rcnt, scnt; 510 int num_cq_events = 0; 511 int sl = 0; 512 int gidx = -1; 513 char gid[33]; 514 515 srand48(getpid() * time(NULL)); 516 517 while (1) { 518 int c; 519 520 static struct option long_options[] = { 521 { .name = "port", .has_arg = 1, .val = 'p' }, 522 { .name = "ib-dev", .has_arg = 1, .val = 'd' }, 523 { .name = "ib-port", .has_arg = 1, .val = 'i' }, 524 { .name = "size", .has_arg = 1, .val = 's' }, 525 { .name = "mtu", .has_arg = 1, .val = 'm' }, 526 { .name = "rx-depth", .has_arg = 1, .val = 'r' }, 527 { .name = "iters", .has_arg = 1, .val = 'n' }, 528 { .name = "sl", .has_arg = 1, .val = 'l' }, 529 { .name = "events", .has_arg = 0, .val = 'e' }, 530 { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 531 { 0 } 532 }; 533 534 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 535 if (c == -1) 536 break; 537 538 switch (c) { 539 case 'p': 540 port = strtol(optarg, NULL, 0); 541 if (port < 0 || port > 65535) { 542 usage(argv[0]); 543 return 1; 544 } 545 break; 546 547 case 'd': 548 ib_devname = strdup(optarg); 549 break; 550 551 case 'i': 552 ib_port = strtol(optarg, NULL, 0); 553 if (ib_port < 0) { 554 usage(argv[0]); 555 return 1; 556 } 557 break; 558 559 case 's': 560 size = strtol(optarg, NULL, 0); 561 break; 562 563 case 'm': 564 mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); 565 if (mtu < 0) { 566 usage(argv[0]); 567 return 1; 568 } 569 break; 570 571 case 'r': 572 rx_depth = strtol(optarg, NULL, 0); 573 break; 574 575 case 'n': 576 iters = strtol(optarg, NULL, 0); 577 break; 578 579 case 'l': 580 sl = strtol(optarg, NULL, 0); 581 break; 582 583 case 'e': 584 ++use_event; 585 break; 586 587 case 'g': 588 gidx = strtol(optarg, NULL, 0); 589 break; 590 591 default: 592 usage(argv[0]); 593 return 1; 594 } 595 } 596 597 if (optind == argc - 1) 598 servername = strdup(argv[optind]); 599 else if (optind < argc) { 600 usage(argv[0]); 601 return 1; 602 } 603 604 page_size = sysconf(_SC_PAGESIZE); 605 606 dev_list = ibv_get_device_list(NULL); 607 if (!dev_list) { 608 perror("Failed to get IB devices list"); 609 return 1; 610 } 611 612 if (!ib_devname) { 613 ib_dev = *dev_list; 614 if (!ib_dev) { 615 fprintf(stderr, "No IB devices found\n"); 616 return 1; 617 } 618 } else { 619 int i; 620 for (i = 0; dev_list[i]; ++i) 621 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) 622 break; 623 ib_dev = dev_list[i]; 624 if (!ib_dev) { 625 fprintf(stderr, "IB device %s not found\n", ib_devname); 626 return 1; 627 } 628 } 629 630 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 631 if (!ctx) 632 return 1; 633 634 routs = pp_post_recv(ctx, ctx->rx_depth); 635 if (routs < ctx->rx_depth) { 636 fprintf(stderr, "Couldn't post receive (%d)\n", routs); 637 return 1; 638 } 639 640 if (use_event) 641 if (ibv_req_notify_cq(ctx->cq, 0)) { 642 fprintf(stderr, "Couldn't request CQ notification\n"); 643 return 1; 644 } 645 646 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 647 fprintf(stderr, "Couldn't get port info\n"); 648 return 1; 649 } 650 651 my_dest.lid = ctx->portinfo.lid; 652 if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 653 fprintf(stderr, "Couldn't get local LID\n"); 654 return 1; 655 } 656 657 if (gidx >= 0) { 658 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 659 fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 660 return 1; 661 } 662 } else 663 memset(&my_dest.gid, 0, sizeof my_dest.gid); 664 665 my_dest.qpn = ctx->qp->qp_num; 666 my_dest.psn = lrand48() & 0xffffff; 667 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 668 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 669 my_dest.lid, my_dest.qpn, my_dest.psn, gid); 670 671 if (servername) 672 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 673 else 674 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 675 676 if (!rem_dest) 677 return 1; 678 679 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 680 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 681 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 682 683 if (servername) 684 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 685 return 1; 686 687 ctx->pending = PINGPONG_RECV_WRID; 688 689 if (servername) { 690 if (pp_post_send(ctx)) { 691 fprintf(stderr, "Couldn't post send\n"); 692 return 1; 693 } 694 ctx->pending |= PINGPONG_SEND_WRID; 695 } 696 697 if (gettimeofday(&start, NULL)) { 698 perror("gettimeofday"); 699 return 1; 700 } 701 702 rcnt = scnt = 0; 703 while (rcnt < iters || scnt < iters) { 704 if (use_event) { 705 struct ibv_cq *ev_cq; 706 void *ev_ctx; 707 708 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 709 fprintf(stderr, "Failed to get cq_event\n"); 710 return 1; 711 } 712 713 ++num_cq_events; 714 715 if (ev_cq != ctx->cq) { 716 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); 717 return 1; 718 } 719 720 if (ibv_req_notify_cq(ctx->cq, 0)) { 721 fprintf(stderr, "Couldn't request CQ notification\n"); 722 return 1; 723 } 724 } 725 726 { 727 struct ibv_wc wc[2]; 728 int ne, i; 729 730 do { 731 ne = ibv_poll_cq(ctx->cq, 2, wc); 732 if (ne < 0) { 733 fprintf(stderr, "poll CQ failed %d\n", ne); 734 return 1; 735 } 736 } while (!use_event && ne < 1); 737 738 for (i = 0; i < ne; ++i) { 739 if (wc[i].status != IBV_WC_SUCCESS) { 740 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", 741 ibv_wc_status_str(wc[i].status), 742 wc[i].status, (int) wc[i].wr_id); 743 return 1; 744 } 745 746 switch ((int) wc[i].wr_id) { 747 case PINGPONG_SEND_WRID: 748 ++scnt; 749 break; 750 751 case PINGPONG_RECV_WRID: 752 if (--routs <= 1) { 753 routs += pp_post_recv(ctx, ctx->rx_depth - routs); 754 if (routs < ctx->rx_depth) { 755 fprintf(stderr, 756 "Couldn't post receive (%d)\n", 757 routs); 758 return 1; 759 } 760 } 761 762 ++rcnt; 763 break; 764 765 default: 766 fprintf(stderr, "Completion for unknown wr_id %d\n", 767 (int) wc[i].wr_id); 768 return 1; 769 } 770 771 ctx->pending &= ~(int) wc[i].wr_id; 772 if (scnt < iters && !ctx->pending) { 773 if (pp_post_send(ctx)) { 774 fprintf(stderr, "Couldn't post send\n"); 775 return 1; 776 } 777 ctx->pending = PINGPONG_RECV_WRID | 778 PINGPONG_SEND_WRID; 779 } 780 } 781 } 782 } 783 784 if (gettimeofday(&end, NULL)) { 785 perror("gettimeofday"); 786 return 1; 787 } 788 789 { 790 float usec = (end.tv_sec - start.tv_sec) * 1000000 + 791 (end.tv_usec - start.tv_usec); 792 long long bytes = (long long) size * iters * 2; 793 794 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", 795 bytes, usec / 1000000., bytes * 8. / usec); 796 printf("%d iters in %.2f seconds = %.2f usec/iter\n", 797 iters, usec / 1000000., usec / iters); 798 } 799 800 ibv_ack_cq_events(ctx->cq, num_cq_events); 801 802 if (pp_close_ctx(ctx)) 803 return 1; 804 805 ibv_free_device_list(dev_list); 806 free(rem_dest); 807 808 return 0; 809} 810