1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff * $Id$ 33219820Sjeff */ 34219820Sjeff 35219820Sjeff#include <stdlib.h> 36219820Sjeff#include <string.h> 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <errno.h> 39219820Sjeff#include <sys/types.h> 40219820Sjeff#include <sys/socket.h> 41219820Sjeff#include <netdb.h> 42219820Sjeff#include <getopt.h> 43219820Sjeff 44219820Sjeff#include <rdma/rdma_cma.h> 45331769Shselasky#include "common.h" 46219820Sjeff 47219820Sjeffstruct cmatest_node { 48219820Sjeff int id; 49219820Sjeff struct rdma_cm_id *cma_id; 50219820Sjeff int connected; 51219820Sjeff struct ibv_pd *pd; 52219820Sjeff struct ibv_cq *cq; 53219820Sjeff struct ibv_mr *mr; 54219820Sjeff struct ibv_ah *ah; 55219820Sjeff uint32_t remote_qpn; 56219820Sjeff uint32_t remote_qkey; 57219820Sjeff void *mem; 58219820Sjeff}; 59219820Sjeff 60219820Sjeffstruct cmatest { 61219820Sjeff struct rdma_event_channel *channel; 62219820Sjeff struct cmatest_node *nodes; 63219820Sjeff int conn_index; 64219820Sjeff int connects_left; 65219820Sjeff 66331769Shselasky struct rdma_addrinfo *rai; 67219820Sjeff}; 68219820Sjeff 69219820Sjeffstatic struct cmatest test; 70219820Sjeffstatic int connections = 1; 71219820Sjeffstatic int message_size = 100; 72219820Sjeffstatic int message_count = 10; 73331769Shselaskystatic const char *port = "7174"; 74219820Sjeffstatic uint8_t set_tos = 0; 75219820Sjeffstatic uint8_t tos; 76219820Sjeffstatic char *dst_addr; 77219820Sjeffstatic char *src_addr; 78331769Shselaskystatic struct rdma_addrinfo hints; 79219820Sjeff 80219820Sjeffstatic int create_message(struct cmatest_node *node) 81219820Sjeff{ 82219820Sjeff if (!message_size) 83219820Sjeff message_count = 0; 84219820Sjeff 85219820Sjeff if (!message_count) 86219820Sjeff return 0; 87219820Sjeff 88219820Sjeff node->mem = malloc(message_size + sizeof(struct ibv_grh)); 89219820Sjeff if (!node->mem) { 90219820Sjeff printf("failed message allocation\n"); 91219820Sjeff return -1; 92219820Sjeff } 93219820Sjeff node->mr = ibv_reg_mr(node->pd, node->mem, 94219820Sjeff message_size + sizeof(struct ibv_grh), 95219820Sjeff IBV_ACCESS_LOCAL_WRITE); 96219820Sjeff if (!node->mr) { 97219820Sjeff printf("failed to reg MR\n"); 98219820Sjeff goto err; 99219820Sjeff } 100219820Sjeff return 0; 101219820Sjefferr: 102219820Sjeff free(node->mem); 103219820Sjeff return -1; 104219820Sjeff} 105219820Sjeff 106219820Sjeffstatic int verify_test_params(struct cmatest_node *node) 107219820Sjeff{ 108219820Sjeff struct ibv_port_attr port_attr; 109219820Sjeff int ret; 110219820Sjeff 111219820Sjeff ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num, 112219820Sjeff &port_attr); 113219820Sjeff if (ret) 114219820Sjeff return ret; 115219820Sjeff 116219820Sjeff if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) { 117219820Sjeff printf("udaddy: message_size %d is larger than active mtu %d\n", 118219820Sjeff message_size, 1 << (port_attr.active_mtu + 7)); 119219820Sjeff return -EINVAL; 120219820Sjeff } 121219820Sjeff 122219820Sjeff return 0; 123219820Sjeff} 124219820Sjeff 125219820Sjeffstatic int init_node(struct cmatest_node *node) 126219820Sjeff{ 127219820Sjeff struct ibv_qp_init_attr init_qp_attr; 128219820Sjeff int cqe, ret; 129219820Sjeff 130219820Sjeff node->pd = ibv_alloc_pd(node->cma_id->verbs); 131219820Sjeff if (!node->pd) { 132219820Sjeff ret = -ENOMEM; 133219820Sjeff printf("udaddy: unable to allocate PD\n"); 134219820Sjeff goto out; 135219820Sjeff } 136219820Sjeff 137219820Sjeff cqe = message_count ? message_count * 2 : 2; 138331769Shselasky node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0); 139219820Sjeff if (!node->cq) { 140219820Sjeff ret = -ENOMEM; 141219820Sjeff printf("udaddy: unable to create CQ\n"); 142219820Sjeff goto out; 143219820Sjeff } 144219820Sjeff 145219820Sjeff memset(&init_qp_attr, 0, sizeof init_qp_attr); 146219820Sjeff init_qp_attr.cap.max_send_wr = message_count ? message_count : 1; 147219820Sjeff init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1; 148219820Sjeff init_qp_attr.cap.max_send_sge = 1; 149219820Sjeff init_qp_attr.cap.max_recv_sge = 1; 150219820Sjeff init_qp_attr.qp_context = node; 151219820Sjeff init_qp_attr.sq_sig_all = 0; 152219820Sjeff init_qp_attr.qp_type = IBV_QPT_UD; 153219820Sjeff init_qp_attr.send_cq = node->cq; 154219820Sjeff init_qp_attr.recv_cq = node->cq; 155219820Sjeff ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr); 156219820Sjeff if (ret) { 157219820Sjeff perror("udaddy: unable to create QP"); 158219820Sjeff goto out; 159219820Sjeff } 160219820Sjeff 161219820Sjeff ret = create_message(node); 162219820Sjeff if (ret) { 163219820Sjeff printf("udaddy: failed to create messages: %d\n", ret); 164219820Sjeff goto out; 165219820Sjeff } 166219820Sjeffout: 167219820Sjeff return ret; 168219820Sjeff} 169219820Sjeff 170219820Sjeffstatic int post_recvs(struct cmatest_node *node) 171219820Sjeff{ 172219820Sjeff struct ibv_recv_wr recv_wr, *recv_failure; 173219820Sjeff struct ibv_sge sge; 174219820Sjeff int i, ret = 0; 175219820Sjeff 176219820Sjeff if (!message_count) 177219820Sjeff return 0; 178219820Sjeff 179219820Sjeff recv_wr.next = NULL; 180219820Sjeff recv_wr.sg_list = &sge; 181219820Sjeff recv_wr.num_sge = 1; 182219820Sjeff recv_wr.wr_id = (uintptr_t) node; 183219820Sjeff 184219820Sjeff sge.length = message_size + sizeof(struct ibv_grh); 185219820Sjeff sge.lkey = node->mr->lkey; 186219820Sjeff sge.addr = (uintptr_t) node->mem; 187219820Sjeff 188219820Sjeff for (i = 0; i < message_count && !ret; i++ ) { 189219820Sjeff ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure); 190219820Sjeff if (ret) { 191219820Sjeff printf("failed to post receives: %d\n", ret); 192219820Sjeff break; 193219820Sjeff } 194219820Sjeff } 195219820Sjeff return ret; 196219820Sjeff} 197219820Sjeff 198219820Sjeffstatic int post_sends(struct cmatest_node *node, int signal_flag) 199219820Sjeff{ 200219820Sjeff struct ibv_send_wr send_wr, *bad_send_wr; 201219820Sjeff struct ibv_sge sge; 202219820Sjeff int i, ret = 0; 203219820Sjeff 204219820Sjeff if (!node->connected || !message_count) 205219820Sjeff return 0; 206219820Sjeff 207219820Sjeff send_wr.next = NULL; 208219820Sjeff send_wr.sg_list = &sge; 209219820Sjeff send_wr.num_sge = 1; 210219820Sjeff send_wr.opcode = IBV_WR_SEND_WITH_IMM; 211219820Sjeff send_wr.send_flags = signal_flag; 212219820Sjeff send_wr.wr_id = (unsigned long)node; 213331769Shselasky send_wr.imm_data = htobe32(node->cma_id->qp->qp_num); 214219820Sjeff 215219820Sjeff send_wr.wr.ud.ah = node->ah; 216219820Sjeff send_wr.wr.ud.remote_qpn = node->remote_qpn; 217219820Sjeff send_wr.wr.ud.remote_qkey = node->remote_qkey; 218219820Sjeff 219219820Sjeff sge.length = message_size; 220219820Sjeff sge.lkey = node->mr->lkey; 221219820Sjeff sge.addr = (uintptr_t) node->mem; 222219820Sjeff 223219820Sjeff for (i = 0; i < message_count && !ret; i++) { 224219820Sjeff ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr); 225219820Sjeff if (ret) 226219820Sjeff printf("failed to post sends: %d\n", ret); 227219820Sjeff } 228219820Sjeff return ret; 229219820Sjeff} 230219820Sjeff 231219820Sjeffstatic void connect_error(void) 232219820Sjeff{ 233219820Sjeff test.connects_left--; 234219820Sjeff} 235219820Sjeff 236219820Sjeffstatic int addr_handler(struct cmatest_node *node) 237219820Sjeff{ 238219820Sjeff int ret; 239219820Sjeff 240219820Sjeff if (set_tos) { 241219820Sjeff ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID, 242219820Sjeff RDMA_OPTION_ID_TOS, &tos, sizeof tos); 243219820Sjeff if (ret) 244219820Sjeff perror("udaddy: set TOS option failed"); 245219820Sjeff } 246219820Sjeff 247219820Sjeff ret = rdma_resolve_route(node->cma_id, 2000); 248219820Sjeff if (ret) { 249219820Sjeff perror("udaddy: resolve route failed"); 250219820Sjeff connect_error(); 251219820Sjeff } 252219820Sjeff return ret; 253219820Sjeff} 254219820Sjeff 255219820Sjeffstatic int route_handler(struct cmatest_node *node) 256219820Sjeff{ 257219820Sjeff struct rdma_conn_param conn_param; 258219820Sjeff int ret; 259219820Sjeff 260219820Sjeff ret = verify_test_params(node); 261219820Sjeff if (ret) 262219820Sjeff goto err; 263219820Sjeff 264219820Sjeff ret = init_node(node); 265219820Sjeff if (ret) 266219820Sjeff goto err; 267219820Sjeff 268219820Sjeff ret = post_recvs(node); 269219820Sjeff if (ret) 270219820Sjeff goto err; 271219820Sjeff 272219820Sjeff memset(&conn_param, 0, sizeof conn_param); 273331769Shselasky conn_param.private_data = test.rai->ai_connect; 274331769Shselasky conn_param.private_data_len = test.rai->ai_connect_len; 275219820Sjeff ret = rdma_connect(node->cma_id, &conn_param); 276219820Sjeff if (ret) { 277219820Sjeff perror("udaddy: failure connecting"); 278219820Sjeff goto err; 279219820Sjeff } 280219820Sjeff return 0; 281219820Sjefferr: 282219820Sjeff connect_error(); 283219820Sjeff return ret; 284219820Sjeff} 285219820Sjeff 286219820Sjeffstatic int connect_handler(struct rdma_cm_id *cma_id) 287219820Sjeff{ 288219820Sjeff struct cmatest_node *node; 289219820Sjeff struct rdma_conn_param conn_param; 290219820Sjeff int ret; 291219820Sjeff 292219820Sjeff if (test.conn_index == connections) { 293219820Sjeff ret = -ENOMEM; 294219820Sjeff goto err1; 295219820Sjeff } 296219820Sjeff node = &test.nodes[test.conn_index++]; 297219820Sjeff 298219820Sjeff node->cma_id = cma_id; 299219820Sjeff cma_id->context = node; 300219820Sjeff 301219820Sjeff ret = verify_test_params(node); 302219820Sjeff if (ret) 303219820Sjeff goto err2; 304219820Sjeff 305219820Sjeff ret = init_node(node); 306219820Sjeff if (ret) 307219820Sjeff goto err2; 308219820Sjeff 309219820Sjeff ret = post_recvs(node); 310219820Sjeff if (ret) 311219820Sjeff goto err2; 312219820Sjeff 313219820Sjeff memset(&conn_param, 0, sizeof conn_param); 314219820Sjeff conn_param.qp_num = node->cma_id->qp->qp_num; 315219820Sjeff ret = rdma_accept(node->cma_id, &conn_param); 316219820Sjeff if (ret) { 317219820Sjeff perror("udaddy: failure accepting"); 318219820Sjeff goto err2; 319219820Sjeff } 320219820Sjeff node->connected = 1; 321219820Sjeff test.connects_left--; 322219820Sjeff return 0; 323219820Sjeff 324219820Sjefferr2: 325219820Sjeff node->cma_id = NULL; 326219820Sjeff connect_error(); 327219820Sjefferr1: 328219820Sjeff printf("udaddy: failing connection request\n"); 329219820Sjeff rdma_reject(cma_id, NULL, 0); 330219820Sjeff return ret; 331219820Sjeff} 332219820Sjeff 333219820Sjeffstatic int resolved_handler(struct cmatest_node *node, 334219820Sjeff struct rdma_cm_event *event) 335219820Sjeff{ 336219820Sjeff node->remote_qpn = event->param.ud.qp_num; 337219820Sjeff node->remote_qkey = event->param.ud.qkey; 338219820Sjeff node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr); 339219820Sjeff if (!node->ah) { 340219820Sjeff printf("udaddy: failure creating address handle\n"); 341219820Sjeff goto err; 342219820Sjeff } 343219820Sjeff 344219820Sjeff node->connected = 1; 345219820Sjeff test.connects_left--; 346219820Sjeff return 0; 347219820Sjefferr: 348219820Sjeff connect_error(); 349219820Sjeff return -1; 350219820Sjeff} 351219820Sjeff 352219820Sjeffstatic int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 353219820Sjeff{ 354219820Sjeff int ret = 0; 355219820Sjeff 356219820Sjeff switch (event->event) { 357219820Sjeff case RDMA_CM_EVENT_ADDR_RESOLVED: 358219820Sjeff ret = addr_handler(cma_id->context); 359219820Sjeff break; 360219820Sjeff case RDMA_CM_EVENT_ROUTE_RESOLVED: 361219820Sjeff ret = route_handler(cma_id->context); 362219820Sjeff break; 363219820Sjeff case RDMA_CM_EVENT_CONNECT_REQUEST: 364219820Sjeff ret = connect_handler(cma_id); 365219820Sjeff break; 366219820Sjeff case RDMA_CM_EVENT_ESTABLISHED: 367219820Sjeff ret = resolved_handler(cma_id->context, event); 368219820Sjeff break; 369219820Sjeff case RDMA_CM_EVENT_ADDR_ERROR: 370219820Sjeff case RDMA_CM_EVENT_ROUTE_ERROR: 371219820Sjeff case RDMA_CM_EVENT_CONNECT_ERROR: 372219820Sjeff case RDMA_CM_EVENT_UNREACHABLE: 373219820Sjeff case RDMA_CM_EVENT_REJECTED: 374219820Sjeff printf("udaddy: event: %s, error: %d\n", 375219820Sjeff rdma_event_str(event->event), event->status); 376219820Sjeff connect_error(); 377219820Sjeff ret = event->status; 378219820Sjeff break; 379219820Sjeff case RDMA_CM_EVENT_DEVICE_REMOVAL: 380219820Sjeff /* Cleanup will occur after test completes. */ 381219820Sjeff break; 382219820Sjeff default: 383219820Sjeff break; 384219820Sjeff } 385219820Sjeff return ret; 386219820Sjeff} 387219820Sjeff 388219820Sjeffstatic void destroy_node(struct cmatest_node *node) 389219820Sjeff{ 390219820Sjeff if (!node->cma_id) 391219820Sjeff return; 392219820Sjeff 393219820Sjeff if (node->ah) 394219820Sjeff ibv_destroy_ah(node->ah); 395219820Sjeff 396219820Sjeff if (node->cma_id->qp) 397219820Sjeff rdma_destroy_qp(node->cma_id); 398219820Sjeff 399219820Sjeff if (node->cq) 400219820Sjeff ibv_destroy_cq(node->cq); 401219820Sjeff 402219820Sjeff if (node->mem) { 403219820Sjeff ibv_dereg_mr(node->mr); 404219820Sjeff free(node->mem); 405219820Sjeff } 406219820Sjeff 407219820Sjeff if (node->pd) 408219820Sjeff ibv_dealloc_pd(node->pd); 409219820Sjeff 410219820Sjeff /* Destroy the RDMA ID after all device resources */ 411219820Sjeff rdma_destroy_id(node->cma_id); 412219820Sjeff} 413219820Sjeff 414219820Sjeffstatic int alloc_nodes(void) 415219820Sjeff{ 416219820Sjeff int ret, i; 417219820Sjeff 418219820Sjeff test.nodes = malloc(sizeof *test.nodes * connections); 419219820Sjeff if (!test.nodes) { 420219820Sjeff printf("udaddy: unable to allocate memory for test nodes\n"); 421219820Sjeff return -ENOMEM; 422219820Sjeff } 423219820Sjeff memset(test.nodes, 0, sizeof *test.nodes * connections); 424219820Sjeff 425219820Sjeff for (i = 0; i < connections; i++) { 426219820Sjeff test.nodes[i].id = i; 427219820Sjeff if (dst_addr) { 428219820Sjeff ret = rdma_create_id(test.channel, 429219820Sjeff &test.nodes[i].cma_id, 430331769Shselasky &test.nodes[i], hints.ai_port_space); 431219820Sjeff if (ret) 432219820Sjeff goto err; 433219820Sjeff } 434219820Sjeff } 435219820Sjeff return 0; 436219820Sjefferr: 437219820Sjeff while (--i >= 0) 438219820Sjeff rdma_destroy_id(test.nodes[i].cma_id); 439219820Sjeff free(test.nodes); 440219820Sjeff return ret; 441219820Sjeff} 442219820Sjeff 443219820Sjeffstatic void destroy_nodes(void) 444219820Sjeff{ 445219820Sjeff int i; 446219820Sjeff 447219820Sjeff for (i = 0; i < connections; i++) 448219820Sjeff destroy_node(&test.nodes[i]); 449219820Sjeff free(test.nodes); 450219820Sjeff} 451219820Sjeff 452219820Sjeffstatic void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc) 453219820Sjeff{ 454219820Sjeff struct ibv_qp_attr attr; 455219820Sjeff struct ibv_qp_init_attr init_attr; 456219820Sjeff 457219820Sjeff node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem, 458219820Sjeff node->cma_id->port_num); 459331769Shselasky node->remote_qpn = be32toh(wc->imm_data); 460219820Sjeff 461219820Sjeff ibv_query_qp(node->cma_id->qp, &attr, IBV_QP_QKEY, &init_attr); 462219820Sjeff node->remote_qkey = attr.qkey; 463219820Sjeff} 464219820Sjeff 465219820Sjeffstatic int poll_cqs(void) 466219820Sjeff{ 467219820Sjeff struct ibv_wc wc[8]; 468219820Sjeff int done, i, ret; 469219820Sjeff 470219820Sjeff for (i = 0; i < connections; i++) { 471219820Sjeff if (!test.nodes[i].connected) 472219820Sjeff continue; 473219820Sjeff 474219820Sjeff for (done = 0; done < message_count; done += ret) { 475219820Sjeff ret = ibv_poll_cq(test.nodes[i].cq, 8, wc); 476219820Sjeff if (ret < 0) { 477219820Sjeff printf("udaddy: failed polling CQ: %d\n", ret); 478219820Sjeff return ret; 479219820Sjeff } 480219820Sjeff 481219820Sjeff if (ret && !test.nodes[i].ah) 482219820Sjeff create_reply_ah(&test.nodes[i], wc); 483219820Sjeff } 484219820Sjeff } 485219820Sjeff return 0; 486219820Sjeff} 487219820Sjeff 488219820Sjeffstatic int connect_events(void) 489219820Sjeff{ 490219820Sjeff struct rdma_cm_event *event; 491219820Sjeff int ret = 0; 492219820Sjeff 493219820Sjeff while (test.connects_left && !ret) { 494219820Sjeff ret = rdma_get_cm_event(test.channel, &event); 495219820Sjeff if (!ret) { 496219820Sjeff ret = cma_handler(event->id, event); 497219820Sjeff rdma_ack_cm_event(event); 498219820Sjeff } 499219820Sjeff } 500219820Sjeff return ret; 501219820Sjeff} 502219820Sjeff 503219820Sjeffstatic int run_server(void) 504219820Sjeff{ 505219820Sjeff struct rdma_cm_id *listen_id; 506219820Sjeff int i, ret; 507219820Sjeff 508219820Sjeff printf("udaddy: starting server\n"); 509331769Shselasky ret = rdma_create_id(test.channel, &listen_id, &test, hints.ai_port_space); 510219820Sjeff if (ret) { 511219820Sjeff perror("udaddy: listen request failed"); 512219820Sjeff return ret; 513219820Sjeff } 514219820Sjeff 515331769Shselasky ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai); 516331769Shselasky if (ret) { 517331769Shselasky printf("udaddy: getrdmaaddr error: %s\n", gai_strerror(ret)); 518331769Shselasky goto out; 519331769Shselasky } 520219820Sjeff 521331769Shselasky ret = rdma_bind_addr(listen_id, test.rai->ai_src_addr); 522219820Sjeff if (ret) { 523219820Sjeff perror("udaddy: bind address failed"); 524331769Shselasky goto out; 525219820Sjeff } 526219820Sjeff 527219820Sjeff ret = rdma_listen(listen_id, 0); 528219820Sjeff if (ret) { 529219820Sjeff perror("udaddy: failure trying to listen"); 530219820Sjeff goto out; 531219820Sjeff } 532219820Sjeff 533219820Sjeff connect_events(); 534219820Sjeff 535219820Sjeff if (message_count) { 536219820Sjeff printf("receiving data transfers\n"); 537219820Sjeff ret = poll_cqs(); 538219820Sjeff if (ret) 539219820Sjeff goto out; 540219820Sjeff 541219820Sjeff printf("sending replies\n"); 542219820Sjeff for (i = 0; i < connections; i++) { 543219820Sjeff ret = post_sends(&test.nodes[i], IBV_SEND_SIGNALED); 544219820Sjeff if (ret) 545219820Sjeff goto out; 546219820Sjeff } 547219820Sjeff 548219820Sjeff ret = poll_cqs(); 549219820Sjeff if (ret) 550219820Sjeff goto out; 551219820Sjeff printf("data transfers complete\n"); 552219820Sjeff } 553219820Sjeffout: 554219820Sjeff rdma_destroy_id(listen_id); 555219820Sjeff return ret; 556219820Sjeff} 557219820Sjeff 558219820Sjeffstatic int run_client(void) 559219820Sjeff{ 560219820Sjeff int i, ret; 561219820Sjeff 562219820Sjeff printf("udaddy: starting client\n"); 563219820Sjeff 564331769Shselasky ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai); 565331769Shselasky if (ret) { 566331769Shselasky printf("udaddy: getaddrinfo error: %s\n", gai_strerror(ret)); 567219820Sjeff return ret; 568331769Shselasky } 569219820Sjeff 570219820Sjeff printf("udaddy: connecting\n"); 571219820Sjeff for (i = 0; i < connections; i++) { 572331769Shselasky ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr, 573331769Shselasky test.rai->ai_dst_addr, 2000); 574219820Sjeff if (ret) { 575219820Sjeff perror("udaddy: failure getting addr"); 576219820Sjeff connect_error(); 577219820Sjeff return ret; 578219820Sjeff } 579219820Sjeff } 580219820Sjeff 581219820Sjeff ret = connect_events(); 582219820Sjeff if (ret) 583219820Sjeff goto out; 584219820Sjeff 585219820Sjeff if (message_count) { 586219820Sjeff printf("initiating data transfers\n"); 587219820Sjeff for (i = 0; i < connections; i++) { 588219820Sjeff ret = post_sends(&test.nodes[i], 0); 589219820Sjeff if (ret) 590219820Sjeff goto out; 591219820Sjeff } 592219820Sjeff printf("receiving data transfers\n"); 593219820Sjeff ret = poll_cqs(); 594219820Sjeff if (ret) 595219820Sjeff goto out; 596219820Sjeff 597219820Sjeff printf("data transfers complete\n"); 598219820Sjeff } 599219820Sjeffout: 600219820Sjeff return ret; 601219820Sjeff} 602219820Sjeff 603219820Sjeffint main(int argc, char **argv) 604219820Sjeff{ 605219820Sjeff int op, ret; 606219820Sjeff 607331769Shselasky hints.ai_port_space = RDMA_PS_UDP; 608331769Shselasky while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:P:f:")) != -1) { 609219820Sjeff switch (op) { 610219820Sjeff case 's': 611219820Sjeff dst_addr = optarg; 612219820Sjeff break; 613219820Sjeff case 'b': 614219820Sjeff src_addr = optarg; 615219820Sjeff break; 616219820Sjeff case 'c': 617219820Sjeff connections = atoi(optarg); 618219820Sjeff break; 619219820Sjeff case 'C': 620219820Sjeff message_count = atoi(optarg); 621219820Sjeff break; 622219820Sjeff case 'S': 623219820Sjeff message_size = atoi(optarg); 624219820Sjeff break; 625219820Sjeff case 't': 626219820Sjeff set_tos = 1; 627331769Shselasky tos = (uint8_t) strtoul(optarg, NULL, 0); 628219820Sjeff break; 629331769Shselasky case 'p': /* for backwards compatibility - use -P */ 630331769Shselasky hints.ai_port_space = strtol(optarg, NULL, 0); 631219820Sjeff break; 632331769Shselasky case 'f': 633331769Shselasky if (!strncasecmp("ip", optarg, 2)) { 634331769Shselasky hints.ai_flags = RAI_NUMERICHOST; 635331769Shselasky } else if (!strncasecmp("gid", optarg, 3)) { 636331769Shselasky hints.ai_flags = RAI_NUMERICHOST | RAI_FAMILY; 637331769Shselasky hints.ai_family = AF_IB; 638331769Shselasky } else if (strncasecmp("name", optarg, 4)) { 639331769Shselasky fprintf(stderr, "Warning: unknown address format\n"); 640331769Shselasky } 641331769Shselasky break; 642331769Shselasky case 'P': 643331769Shselasky if (!strncasecmp("ipoib", optarg, 5)) { 644331769Shselasky hints.ai_port_space = RDMA_PS_IPOIB; 645331769Shselasky } else if (strncasecmp("udp", optarg, 3)) { 646331769Shselasky fprintf(stderr, "Warning: unknown port space format\n"); 647331769Shselasky } 648331769Shselasky break; 649219820Sjeff default: 650219820Sjeff printf("usage: %s\n", argv[0]); 651219820Sjeff printf("\t[-s server_address]\n"); 652219820Sjeff printf("\t[-b bind_address]\n"); 653331769Shselasky printf("\t[-f address_format]\n"); 654331769Shselasky printf("\t name, ip, ipv6, or gid\n"); 655331769Shselasky printf("\t[-P port_space]\n"); 656331769Shselasky printf("\t udp or ipoib\n"); 657219820Sjeff printf("\t[-c connections]\n"); 658219820Sjeff printf("\t[-C message_count]\n"); 659219820Sjeff printf("\t[-S message_size]\n"); 660219820Sjeff printf("\t[-t type_of_service]\n"); 661219820Sjeff printf("\t[-p port_space - %#x for UDP (default), " 662219820Sjeff "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB); 663219820Sjeff exit(1); 664219820Sjeff } 665219820Sjeff } 666219820Sjeff 667219820Sjeff test.connects_left = connections; 668219820Sjeff 669219820Sjeff test.channel = rdma_create_event_channel(); 670219820Sjeff if (!test.channel) { 671219820Sjeff perror("failed to create event channel"); 672219820Sjeff exit(1); 673219820Sjeff } 674219820Sjeff 675219820Sjeff if (alloc_nodes()) 676219820Sjeff exit(1); 677219820Sjeff 678331769Shselasky if (dst_addr) { 679219820Sjeff ret = run_client(); 680331769Shselasky } else { 681331769Shselasky hints.ai_flags |= RAI_PASSIVE; 682219820Sjeff ret = run_server(); 683331769Shselasky } 684219820Sjeff 685219820Sjeff printf("test complete\n"); 686219820Sjeff destroy_nodes(); 687219820Sjeff rdma_destroy_event_channel(test.channel); 688331769Shselasky if (test.rai) 689331769Shselasky rdma_freeaddrinfo(test.rai); 690219820Sjeff 691219820Sjeff printf("return status %d\n", ret); 692219820Sjeff return ret; 693219820Sjeff} 694