1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff * $Id$ 33219820Sjeff */ 34219820Sjeff 35219820Sjeff#include <stdlib.h> 36219820Sjeff#include <string.h> 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <errno.h> 39219820Sjeff#include <sys/types.h> 40219820Sjeff#include <arpa/inet.h> 41219820Sjeff#include <sys/socket.h> 42219820Sjeff#include <netdb.h> 43219820Sjeff#include <unistd.h> 44219820Sjeff#include <getopt.h> 45219820Sjeff 46219820Sjeff#include <rdma/rdma_cma.h> 47331769Shselasky#include <infiniband/ib.h> 48219820Sjeff 49219820Sjeffstruct cmatest_node { 50219820Sjeff int id; 51219820Sjeff struct rdma_cm_id *cma_id; 52219820Sjeff int connected; 53219820Sjeff struct ibv_pd *pd; 54219820Sjeff struct ibv_cq *cq; 55219820Sjeff struct ibv_mr *mr; 56219820Sjeff struct ibv_ah *ah; 57219820Sjeff uint32_t remote_qpn; 58219820Sjeff uint32_t remote_qkey; 59219820Sjeff void *mem; 60219820Sjeff}; 61219820Sjeff 62219820Sjeffstruct cmatest { 63219820Sjeff struct rdma_event_channel *channel; 64219820Sjeff pthread_t cmathread; 65219820Sjeff struct cmatest_node *nodes; 66219820Sjeff int conn_index; 67219820Sjeff int connects_left; 68219820Sjeff 69331769Shselasky struct sockaddr_storage dst_in; 70219820Sjeff struct sockaddr *dst_addr; 71331769Shselasky struct sockaddr_storage src_in; 72219820Sjeff struct sockaddr *src_addr; 73219820Sjeff}; 74219820Sjeff 75219820Sjeffstatic struct cmatest test; 76219820Sjeffstatic int connections = 1; 77219820Sjeffstatic int message_size = 100; 78219820Sjeffstatic int message_count = 10; 79219820Sjeffstatic int is_sender; 80219820Sjeffstatic int unmapped_addr; 81219820Sjeffstatic char *dst_addr; 82219820Sjeffstatic char *src_addr; 83219820Sjeffstatic enum rdma_port_space port_space = RDMA_PS_UDP; 84219820Sjeff 85219820Sjeffstatic int create_message(struct cmatest_node *node) 86219820Sjeff{ 87219820Sjeff if (!message_size) 88219820Sjeff message_count = 0; 89219820Sjeff 90219820Sjeff if (!message_count) 91219820Sjeff return 0; 92219820Sjeff 93219820Sjeff node->mem = malloc(message_size + sizeof(struct ibv_grh)); 94219820Sjeff if (!node->mem) { 95219820Sjeff printf("failed message allocation\n"); 96219820Sjeff return -1; 97219820Sjeff } 98219820Sjeff node->mr = ibv_reg_mr(node->pd, node->mem, 99219820Sjeff message_size + sizeof(struct ibv_grh), 100219820Sjeff IBV_ACCESS_LOCAL_WRITE); 101219820Sjeff if (!node->mr) { 102219820Sjeff printf("failed to reg MR\n"); 103219820Sjeff goto err; 104219820Sjeff } 105219820Sjeff return 0; 106219820Sjefferr: 107219820Sjeff free(node->mem); 108219820Sjeff return -1; 109219820Sjeff} 110219820Sjeff 111219820Sjeffstatic int verify_test_params(struct cmatest_node *node) 112219820Sjeff{ 113219820Sjeff struct ibv_port_attr port_attr; 114219820Sjeff int ret; 115219820Sjeff 116219820Sjeff ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num, 117219820Sjeff &port_attr); 118219820Sjeff if (ret) 119219820Sjeff return ret; 120219820Sjeff 121219820Sjeff if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) { 122219820Sjeff printf("mckey: message_size %d is larger than active mtu %d\n", 123219820Sjeff message_size, 1 << (port_attr.active_mtu + 7)); 124219820Sjeff return -EINVAL; 125219820Sjeff } 126219820Sjeff 127219820Sjeff return 0; 128219820Sjeff} 129219820Sjeff 130219820Sjeffstatic int init_node(struct cmatest_node *node) 131219820Sjeff{ 132219820Sjeff struct ibv_qp_init_attr init_qp_attr; 133219820Sjeff int cqe, ret; 134219820Sjeff 135219820Sjeff node->pd = ibv_alloc_pd(node->cma_id->verbs); 136219820Sjeff if (!node->pd) { 137219820Sjeff ret = -ENOMEM; 138219820Sjeff printf("mckey: unable to allocate PD\n"); 139219820Sjeff goto out; 140219820Sjeff } 141219820Sjeff 142219820Sjeff cqe = message_count ? message_count * 2 : 2; 143331769Shselasky node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0); 144219820Sjeff if (!node->cq) { 145219820Sjeff ret = -ENOMEM; 146219820Sjeff printf("mckey: unable to create CQ\n"); 147219820Sjeff goto out; 148219820Sjeff } 149219820Sjeff 150219820Sjeff memset(&init_qp_attr, 0, sizeof init_qp_attr); 151219820Sjeff init_qp_attr.cap.max_send_wr = message_count ? message_count : 1; 152219820Sjeff init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1; 153219820Sjeff init_qp_attr.cap.max_send_sge = 1; 154219820Sjeff init_qp_attr.cap.max_recv_sge = 1; 155219820Sjeff init_qp_attr.qp_context = node; 156219820Sjeff init_qp_attr.sq_sig_all = 0; 157219820Sjeff init_qp_attr.qp_type = IBV_QPT_UD; 158219820Sjeff init_qp_attr.send_cq = node->cq; 159219820Sjeff init_qp_attr.recv_cq = node->cq; 160219820Sjeff ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr); 161219820Sjeff if (ret) { 162219820Sjeff perror("mckey: unable to create QP"); 163219820Sjeff goto out; 164219820Sjeff } 165219820Sjeff 166219820Sjeff ret = create_message(node); 167219820Sjeff if (ret) { 168219820Sjeff printf("mckey: failed to create messages: %d\n", ret); 169219820Sjeff goto out; 170219820Sjeff } 171219820Sjeffout: 172219820Sjeff return ret; 173219820Sjeff} 174219820Sjeff 175219820Sjeffstatic int post_recvs(struct cmatest_node *node) 176219820Sjeff{ 177219820Sjeff struct ibv_recv_wr recv_wr, *recv_failure; 178219820Sjeff struct ibv_sge sge; 179219820Sjeff int i, ret = 0; 180219820Sjeff 181219820Sjeff if (!message_count) 182219820Sjeff return 0; 183219820Sjeff 184219820Sjeff recv_wr.next = NULL; 185219820Sjeff recv_wr.sg_list = &sge; 186219820Sjeff recv_wr.num_sge = 1; 187219820Sjeff recv_wr.wr_id = (uintptr_t) node; 188219820Sjeff 189219820Sjeff sge.length = message_size + sizeof(struct ibv_grh); 190219820Sjeff sge.lkey = node->mr->lkey; 191219820Sjeff sge.addr = (uintptr_t) node->mem; 192219820Sjeff 193219820Sjeff for (i = 0; i < message_count && !ret; i++ ) { 194219820Sjeff ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure); 195219820Sjeff if (ret) { 196219820Sjeff printf("failed to post receives: %d\n", ret); 197219820Sjeff break; 198219820Sjeff } 199219820Sjeff } 200219820Sjeff return ret; 201219820Sjeff} 202219820Sjeff 203219820Sjeffstatic int post_sends(struct cmatest_node *node, int signal_flag) 204219820Sjeff{ 205219820Sjeff struct ibv_send_wr send_wr, *bad_send_wr; 206219820Sjeff struct ibv_sge sge; 207219820Sjeff int i, ret = 0; 208219820Sjeff 209219820Sjeff if (!node->connected || !message_count) 210219820Sjeff return 0; 211219820Sjeff 212219820Sjeff send_wr.next = NULL; 213219820Sjeff send_wr.sg_list = &sge; 214219820Sjeff send_wr.num_sge = 1; 215219820Sjeff send_wr.opcode = IBV_WR_SEND_WITH_IMM; 216219820Sjeff send_wr.send_flags = signal_flag; 217219820Sjeff send_wr.wr_id = (unsigned long)node; 218331769Shselasky send_wr.imm_data = htobe32(node->cma_id->qp->qp_num); 219219820Sjeff 220219820Sjeff send_wr.wr.ud.ah = node->ah; 221219820Sjeff send_wr.wr.ud.remote_qpn = node->remote_qpn; 222219820Sjeff send_wr.wr.ud.remote_qkey = node->remote_qkey; 223219820Sjeff 224219820Sjeff sge.length = message_size; 225219820Sjeff sge.lkey = node->mr->lkey; 226219820Sjeff sge.addr = (uintptr_t) node->mem; 227219820Sjeff 228219820Sjeff for (i = 0; i < message_count && !ret; i++) { 229219820Sjeff ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr); 230219820Sjeff if (ret) 231219820Sjeff printf("failed to post sends: %d\n", ret); 232219820Sjeff } 233219820Sjeff return ret; 234219820Sjeff} 235219820Sjeff 236219820Sjeffstatic void connect_error(void) 237219820Sjeff{ 238219820Sjeff test.connects_left--; 239219820Sjeff} 240219820Sjeff 241219820Sjeffstatic int addr_handler(struct cmatest_node *node) 242219820Sjeff{ 243219820Sjeff int ret; 244219820Sjeff 245219820Sjeff ret = verify_test_params(node); 246219820Sjeff if (ret) 247219820Sjeff goto err; 248219820Sjeff 249219820Sjeff ret = init_node(node); 250219820Sjeff if (ret) 251219820Sjeff goto err; 252219820Sjeff 253219820Sjeff if (!is_sender) { 254219820Sjeff ret = post_recvs(node); 255219820Sjeff if (ret) 256219820Sjeff goto err; 257219820Sjeff } 258219820Sjeff 259219820Sjeff ret = rdma_join_multicast(node->cma_id, test.dst_addr, node); 260219820Sjeff if (ret) { 261219820Sjeff perror("mckey: failure joining"); 262219820Sjeff goto err; 263219820Sjeff } 264219820Sjeff return 0; 265219820Sjefferr: 266219820Sjeff connect_error(); 267219820Sjeff return ret; 268219820Sjeff} 269219820Sjeff 270219820Sjeffstatic int join_handler(struct cmatest_node *node, 271219820Sjeff struct rdma_ud_param *param) 272219820Sjeff{ 273219820Sjeff char buf[40]; 274219820Sjeff 275219820Sjeff inet_ntop(AF_INET6, param->ah_attr.grh.dgid.raw, buf, 40); 276219820Sjeff printf("mckey: joined dgid: %s mlid 0x%x sl %d\n", buf, 277219820Sjeff param->ah_attr.dlid, param->ah_attr.sl); 278219820Sjeff 279219820Sjeff node->remote_qpn = param->qp_num; 280219820Sjeff node->remote_qkey = param->qkey; 281219820Sjeff node->ah = ibv_create_ah(node->pd, ¶m->ah_attr); 282219820Sjeff if (!node->ah) { 283219820Sjeff printf("mckey: failure creating address handle\n"); 284219820Sjeff goto err; 285219820Sjeff } 286219820Sjeff 287219820Sjeff node->connected = 1; 288219820Sjeff test.connects_left--; 289219820Sjeff return 0; 290219820Sjefferr: 291219820Sjeff connect_error(); 292219820Sjeff return -1; 293219820Sjeff} 294219820Sjeff 295219820Sjeffstatic int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 296219820Sjeff{ 297219820Sjeff int ret = 0; 298219820Sjeff 299219820Sjeff switch (event->event) { 300219820Sjeff case RDMA_CM_EVENT_ADDR_RESOLVED: 301219820Sjeff ret = addr_handler(cma_id->context); 302219820Sjeff break; 303219820Sjeff case RDMA_CM_EVENT_MULTICAST_JOIN: 304219820Sjeff ret = join_handler(cma_id->context, &event->param.ud); 305219820Sjeff break; 306219820Sjeff case RDMA_CM_EVENT_ADDR_ERROR: 307219820Sjeff case RDMA_CM_EVENT_ROUTE_ERROR: 308219820Sjeff case RDMA_CM_EVENT_MULTICAST_ERROR: 309219820Sjeff printf("mckey: event: %s, error: %d\n", 310219820Sjeff rdma_event_str(event->event), event->status); 311219820Sjeff connect_error(); 312219820Sjeff ret = event->status; 313219820Sjeff break; 314219820Sjeff case RDMA_CM_EVENT_DEVICE_REMOVAL: 315219820Sjeff /* Cleanup will occur after test completes. */ 316219820Sjeff break; 317219820Sjeff default: 318219820Sjeff break; 319219820Sjeff } 320219820Sjeff return ret; 321219820Sjeff} 322219820Sjeff 323219820Sjeffstatic void *cma_thread(void *arg) 324219820Sjeff{ 325219820Sjeff struct rdma_cm_event *event; 326219820Sjeff int ret; 327219820Sjeff 328219820Sjeff while (1) { 329219820Sjeff ret = rdma_get_cm_event(test.channel, &event); 330219820Sjeff if (ret) { 331219820Sjeff perror("rdma_get_cm_event"); 332219820Sjeff break; 333219820Sjeff } 334219820Sjeff 335219820Sjeff switch (event->event) { 336219820Sjeff case RDMA_CM_EVENT_MULTICAST_ERROR: 337219820Sjeff case RDMA_CM_EVENT_ADDR_CHANGE: 338219820Sjeff printf("mckey: event: %s, status: %d\n", 339219820Sjeff rdma_event_str(event->event), event->status); 340219820Sjeff break; 341219820Sjeff default: 342219820Sjeff break; 343219820Sjeff } 344219820Sjeff 345219820Sjeff rdma_ack_cm_event(event); 346219820Sjeff } 347219820Sjeff return NULL; 348219820Sjeff} 349219820Sjeff 350219820Sjeffstatic void destroy_node(struct cmatest_node *node) 351219820Sjeff{ 352219820Sjeff if (!node->cma_id) 353219820Sjeff return; 354219820Sjeff 355219820Sjeff if (node->ah) 356219820Sjeff ibv_destroy_ah(node->ah); 357219820Sjeff 358219820Sjeff if (node->cma_id->qp) 359219820Sjeff rdma_destroy_qp(node->cma_id); 360219820Sjeff 361219820Sjeff if (node->cq) 362219820Sjeff ibv_destroy_cq(node->cq); 363219820Sjeff 364219820Sjeff if (node->mem) { 365219820Sjeff ibv_dereg_mr(node->mr); 366219820Sjeff free(node->mem); 367219820Sjeff } 368219820Sjeff 369219820Sjeff if (node->pd) 370219820Sjeff ibv_dealloc_pd(node->pd); 371219820Sjeff 372219820Sjeff /* Destroy the RDMA ID after all device resources */ 373219820Sjeff rdma_destroy_id(node->cma_id); 374219820Sjeff} 375219820Sjeff 376219820Sjeffstatic int alloc_nodes(void) 377219820Sjeff{ 378219820Sjeff int ret, i; 379219820Sjeff 380219820Sjeff test.nodes = malloc(sizeof *test.nodes * connections); 381219820Sjeff if (!test.nodes) { 382219820Sjeff printf("mckey: unable to allocate memory for test nodes\n"); 383219820Sjeff return -ENOMEM; 384219820Sjeff } 385219820Sjeff memset(test.nodes, 0, sizeof *test.nodes * connections); 386219820Sjeff 387219820Sjeff for (i = 0; i < connections; i++) { 388219820Sjeff test.nodes[i].id = i; 389219820Sjeff ret = rdma_create_id(test.channel, &test.nodes[i].cma_id, 390219820Sjeff &test.nodes[i], port_space); 391219820Sjeff if (ret) 392219820Sjeff goto err; 393219820Sjeff } 394219820Sjeff return 0; 395219820Sjefferr: 396219820Sjeff while (--i >= 0) 397219820Sjeff rdma_destroy_id(test.nodes[i].cma_id); 398219820Sjeff free(test.nodes); 399219820Sjeff return ret; 400219820Sjeff} 401219820Sjeff 402219820Sjeffstatic void destroy_nodes(void) 403219820Sjeff{ 404219820Sjeff int i; 405219820Sjeff 406219820Sjeff for (i = 0; i < connections; i++) 407219820Sjeff destroy_node(&test.nodes[i]); 408219820Sjeff free(test.nodes); 409219820Sjeff} 410219820Sjeff 411219820Sjeffstatic int poll_cqs(void) 412219820Sjeff{ 413219820Sjeff struct ibv_wc wc[8]; 414219820Sjeff int done, i, ret; 415219820Sjeff 416219820Sjeff for (i = 0; i < connections; i++) { 417219820Sjeff if (!test.nodes[i].connected) 418219820Sjeff continue; 419219820Sjeff 420219820Sjeff for (done = 0; done < message_count; done += ret) { 421219820Sjeff ret = ibv_poll_cq(test.nodes[i].cq, 8, wc); 422219820Sjeff if (ret < 0) { 423219820Sjeff printf("mckey: failed polling CQ: %d\n", ret); 424219820Sjeff return ret; 425219820Sjeff } 426219820Sjeff } 427219820Sjeff } 428219820Sjeff return 0; 429219820Sjeff} 430219820Sjeff 431219820Sjeffstatic int connect_events(void) 432219820Sjeff{ 433219820Sjeff struct rdma_cm_event *event; 434219820Sjeff int ret = 0; 435219820Sjeff 436219820Sjeff while (test.connects_left && !ret) { 437219820Sjeff ret = rdma_get_cm_event(test.channel, &event); 438219820Sjeff if (!ret) { 439219820Sjeff ret = cma_handler(event->id, event); 440219820Sjeff rdma_ack_cm_event(event); 441219820Sjeff } 442219820Sjeff } 443219820Sjeff return ret; 444219820Sjeff} 445219820Sjeff 446219820Sjeffstatic int get_addr(char *dst, struct sockaddr *addr) 447219820Sjeff{ 448219820Sjeff struct addrinfo *res; 449219820Sjeff int ret; 450219820Sjeff 451219820Sjeff ret = getaddrinfo(dst, NULL, NULL, &res); 452219820Sjeff if (ret) { 453331769Shselasky printf("getaddrinfo failed (%s) - invalid hostname or IP address\n", gai_strerror(ret)); 454219820Sjeff return ret; 455219820Sjeff } 456219820Sjeff 457219820Sjeff memcpy(addr, res->ai_addr, res->ai_addrlen); 458219820Sjeff freeaddrinfo(res); 459219820Sjeff return ret; 460219820Sjeff} 461219820Sjeff 462331769Shselaskystatic int get_dst_addr(char *dst, struct sockaddr *addr) 463331769Shselasky{ 464331769Shselasky struct sockaddr_ib *sib; 465331769Shselasky 466331769Shselasky if (!unmapped_addr) 467331769Shselasky return get_addr(dst, addr); 468331769Shselasky 469331769Shselasky sib = (struct sockaddr_ib *) addr; 470331769Shselasky memset(sib, 0, sizeof *sib); 471331769Shselasky sib->sib_family = AF_IB; 472331769Shselasky inet_pton(AF_INET6, dst, &sib->sib_addr); 473331769Shselasky return 0; 474331769Shselasky} 475331769Shselasky 476219820Sjeffstatic int run(void) 477219820Sjeff{ 478335614Sslavash int i, ret, err; 479219820Sjeff 480219820Sjeff printf("mckey: starting %s\n", is_sender ? "client" : "server"); 481219820Sjeff if (src_addr) { 482219820Sjeff ret = get_addr(src_addr, (struct sockaddr *) &test.src_in); 483219820Sjeff if (ret) 484219820Sjeff return ret; 485219820Sjeff } 486219820Sjeff 487331769Shselasky ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in); 488219820Sjeff if (ret) 489219820Sjeff return ret; 490219820Sjeff 491219820Sjeff printf("mckey: joining\n"); 492219820Sjeff for (i = 0; i < connections; i++) { 493219820Sjeff if (src_addr) { 494219820Sjeff ret = rdma_bind_addr(test.nodes[i].cma_id, 495219820Sjeff test.src_addr); 496219820Sjeff if (ret) { 497219820Sjeff perror("mckey: addr bind failure"); 498219820Sjeff connect_error(); 499219820Sjeff return ret; 500219820Sjeff } 501219820Sjeff } 502219820Sjeff 503219820Sjeff if (unmapped_addr) 504219820Sjeff ret = addr_handler(&test.nodes[i]); 505219820Sjeff else 506219820Sjeff ret = rdma_resolve_addr(test.nodes[i].cma_id, 507219820Sjeff test.src_addr, test.dst_addr, 508219820Sjeff 2000); 509219820Sjeff if (ret) { 510219820Sjeff perror("mckey: resolve addr failure"); 511219820Sjeff connect_error(); 512219820Sjeff return ret; 513219820Sjeff } 514219820Sjeff } 515219820Sjeff 516219820Sjeff ret = connect_events(); 517219820Sjeff if (ret) 518219820Sjeff goto out; 519219820Sjeff 520219820Sjeff pthread_create(&test.cmathread, NULL, cma_thread, NULL); 521219820Sjeff 522219820Sjeff /* 523219820Sjeff * Pause to give SM chance to configure switches. We don't want to 524219820Sjeff * handle reliability issue in this simple test program. 525219820Sjeff */ 526219820Sjeff sleep(3); 527219820Sjeff 528219820Sjeff if (message_count) { 529219820Sjeff if (is_sender) { 530219820Sjeff printf("initiating data transfers\n"); 531219820Sjeff for (i = 0; i < connections; i++) { 532219820Sjeff ret = post_sends(&test.nodes[i], 0); 533219820Sjeff if (ret) 534219820Sjeff goto out; 535219820Sjeff } 536219820Sjeff } else { 537219820Sjeff printf("receiving data transfers\n"); 538219820Sjeff ret = poll_cqs(); 539219820Sjeff if (ret) 540219820Sjeff goto out; 541219820Sjeff } 542219820Sjeff printf("data transfers complete\n"); 543219820Sjeff } 544219820Sjeffout: 545219820Sjeff for (i = 0; i < connections; i++) { 546335614Sslavash err = rdma_leave_multicast(test.nodes[i].cma_id, 547219820Sjeff test.dst_addr); 548335614Sslavash if (err) { 549219820Sjeff perror("mckey: failure leaving"); 550335614Sslavash ret = err; 551335614Sslavash } 552219820Sjeff } 553219820Sjeff return ret; 554219820Sjeff} 555219820Sjeff 556219820Sjeffint main(int argc, char **argv) 557219820Sjeff{ 558219820Sjeff int op, ret; 559219820Sjeff 560219820Sjeff 561219820Sjeff while ((op = getopt(argc, argv, "m:M:sb:c:C:S:p:")) != -1) { 562219820Sjeff switch (op) { 563219820Sjeff case 'm': 564219820Sjeff dst_addr = optarg; 565219820Sjeff break; 566219820Sjeff case 'M': 567219820Sjeff unmapped_addr = 1; 568219820Sjeff dst_addr = optarg; 569219820Sjeff break; 570219820Sjeff case 's': 571219820Sjeff is_sender = 1; 572219820Sjeff break; 573219820Sjeff case 'b': 574219820Sjeff src_addr = optarg; 575219820Sjeff test.src_addr = (struct sockaddr *) &test.src_in; 576219820Sjeff break; 577219820Sjeff case 'c': 578219820Sjeff connections = atoi(optarg); 579219820Sjeff break; 580219820Sjeff case 'C': 581219820Sjeff message_count = atoi(optarg); 582219820Sjeff break; 583219820Sjeff case 'S': 584219820Sjeff message_size = atoi(optarg); 585219820Sjeff break; 586219820Sjeff case 'p': 587219820Sjeff port_space = strtol(optarg, NULL, 0); 588219820Sjeff break; 589219820Sjeff default: 590219820Sjeff printf("usage: %s\n", argv[0]); 591219820Sjeff printf("\t-m multicast_address\n"); 592219820Sjeff printf("\t[-M unmapped_multicast_address]\n" 593219820Sjeff "\t replaces -m and requires -b\n"); 594219820Sjeff printf("\t[-s(ender)]\n"); 595219820Sjeff printf("\t[-b bind_address]\n"); 596219820Sjeff printf("\t[-c connections]\n"); 597219820Sjeff printf("\t[-C message_count]\n"); 598219820Sjeff printf("\t[-S message_size]\n"); 599219820Sjeff printf("\t[-p port_space - %#x for UDP (default), " 600219820Sjeff "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB); 601219820Sjeff exit(1); 602219820Sjeff } 603219820Sjeff } 604219820Sjeff 605219820Sjeff if (unmapped_addr && !src_addr) { 606219820Sjeff printf("unmapped multicast address requires binding " 607219820Sjeff "to source address\n"); 608219820Sjeff exit(1); 609219820Sjeff } 610219820Sjeff 611219820Sjeff test.dst_addr = (struct sockaddr *) &test.dst_in; 612219820Sjeff test.connects_left = connections; 613219820Sjeff 614219820Sjeff test.channel = rdma_create_event_channel(); 615219820Sjeff if (!test.channel) { 616219820Sjeff perror("failed to create event channel"); 617219820Sjeff exit(1); 618219820Sjeff } 619219820Sjeff 620219820Sjeff if (alloc_nodes()) 621219820Sjeff exit(1); 622219820Sjeff 623219820Sjeff ret = run(); 624219820Sjeff 625219820Sjeff printf("test complete\n"); 626219820Sjeff destroy_nodes(); 627219820Sjeff rdma_destroy_event_channel(test.channel); 628219820Sjeff 629219820Sjeff printf("return status %d\n", ret); 630219820Sjeff return ret; 631219820Sjeff} 632