udaddy.c revision 219820
1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff * $Id$ 33219820Sjeff */ 34219820Sjeff 35219820Sjeff#include <stdlib.h> 36219820Sjeff#include <string.h> 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <errno.h> 39219820Sjeff#include <sys/types.h> 40219820Sjeff#include <netinet/in.h> 41219820Sjeff#include <sys/socket.h> 42219820Sjeff#include <netdb.h> 43219820Sjeff#include <byteswap.h> 44219820Sjeff#include <getopt.h> 45219820Sjeff 46219820Sjeff#include <rdma/rdma_cma.h> 47219820Sjeff 48219820Sjeffstruct cmatest_node { 49219820Sjeff int id; 50219820Sjeff struct rdma_cm_id *cma_id; 51219820Sjeff int connected; 52219820Sjeff struct ibv_pd *pd; 53219820Sjeff struct ibv_cq *cq; 54219820Sjeff struct ibv_mr *mr; 55219820Sjeff struct ibv_ah *ah; 56219820Sjeff uint32_t remote_qpn; 57219820Sjeff uint32_t remote_qkey; 58219820Sjeff void *mem; 59219820Sjeff}; 60219820Sjeff 61219820Sjeffstruct cmatest { 62219820Sjeff struct rdma_event_channel *channel; 63219820Sjeff struct cmatest_node *nodes; 64219820Sjeff int conn_index; 65219820Sjeff int connects_left; 66219820Sjeff 67219820Sjeff struct sockaddr_in dst_in; 68219820Sjeff struct sockaddr *dst_addr; 69219820Sjeff struct sockaddr_in src_in; 70219820Sjeff struct sockaddr *src_addr; 71219820Sjeff}; 72219820Sjeff 73219820Sjeffstatic struct cmatest test; 74219820Sjeffstatic int connections = 1; 75219820Sjeffstatic int message_size = 100; 76219820Sjeffstatic int message_count = 10; 77219820Sjeffstatic uint16_t port = 7174; 78219820Sjeffstatic uint8_t set_tos = 0; 79219820Sjeffstatic uint8_t tos; 80219820Sjeffstatic char *dst_addr; 81219820Sjeffstatic char *src_addr; 82219820Sjeffstatic enum rdma_port_space port_space = RDMA_PS_UDP; 83219820Sjeff 84219820Sjeffstatic int create_message(struct cmatest_node *node) 85219820Sjeff{ 86219820Sjeff if (!message_size) 87219820Sjeff message_count = 0; 88219820Sjeff 89219820Sjeff if (!message_count) 90219820Sjeff return 0; 91219820Sjeff 92219820Sjeff node->mem = malloc(message_size + sizeof(struct ibv_grh)); 93219820Sjeff if (!node->mem) { 94219820Sjeff printf("failed message allocation\n"); 95219820Sjeff return -1; 96219820Sjeff } 97219820Sjeff node->mr = ibv_reg_mr(node->pd, node->mem, 98219820Sjeff message_size + sizeof(struct ibv_grh), 99219820Sjeff IBV_ACCESS_LOCAL_WRITE); 100219820Sjeff if (!node->mr) { 101219820Sjeff printf("failed to reg MR\n"); 102219820Sjeff goto err; 103219820Sjeff } 104219820Sjeff return 0; 105219820Sjefferr: 106219820Sjeff free(node->mem); 107219820Sjeff return -1; 108219820Sjeff} 109219820Sjeff 110219820Sjeffstatic int verify_test_params(struct cmatest_node *node) 111219820Sjeff{ 112219820Sjeff struct ibv_port_attr port_attr; 113219820Sjeff int ret; 114219820Sjeff 115219820Sjeff ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num, 116219820Sjeff &port_attr); 117219820Sjeff if (ret) 118219820Sjeff return ret; 119219820Sjeff 120219820Sjeff if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) { 121219820Sjeff printf("udaddy: message_size %d is larger than active mtu %d\n", 122219820Sjeff message_size, 1 << (port_attr.active_mtu + 7)); 123219820Sjeff return -EINVAL; 124219820Sjeff } 125219820Sjeff 126219820Sjeff return 0; 127219820Sjeff} 128219820Sjeff 129219820Sjeffstatic int init_node(struct cmatest_node *node) 130219820Sjeff{ 131219820Sjeff struct ibv_qp_init_attr init_qp_attr; 132219820Sjeff int cqe, ret; 133219820Sjeff 134219820Sjeff node->pd = ibv_alloc_pd(node->cma_id->verbs); 135219820Sjeff if (!node->pd) { 136219820Sjeff ret = -ENOMEM; 137219820Sjeff printf("udaddy: unable to allocate PD\n"); 138219820Sjeff goto out; 139219820Sjeff } 140219820Sjeff 141219820Sjeff cqe = message_count ? message_count * 2 : 2; 142219820Sjeff node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0); 143219820Sjeff if (!node->cq) { 144219820Sjeff ret = -ENOMEM; 145219820Sjeff printf("udaddy: unable to create CQ\n"); 146219820Sjeff goto out; 147219820Sjeff } 148219820Sjeff 149219820Sjeff memset(&init_qp_attr, 0, sizeof init_qp_attr); 150219820Sjeff init_qp_attr.cap.max_send_wr = message_count ? message_count : 1; 151219820Sjeff init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1; 152219820Sjeff init_qp_attr.cap.max_send_sge = 1; 153219820Sjeff init_qp_attr.cap.max_recv_sge = 1; 154219820Sjeff init_qp_attr.qp_context = node; 155219820Sjeff init_qp_attr.sq_sig_all = 0; 156219820Sjeff init_qp_attr.qp_type = IBV_QPT_UD; 157219820Sjeff init_qp_attr.send_cq = node->cq; 158219820Sjeff init_qp_attr.recv_cq = node->cq; 159219820Sjeff ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr); 160219820Sjeff if (ret) { 161219820Sjeff perror("udaddy: unable to create QP"); 162219820Sjeff goto out; 163219820Sjeff } 164219820Sjeff 165219820Sjeff ret = create_message(node); 166219820Sjeff if (ret) { 167219820Sjeff printf("udaddy: failed to create messages: %d\n", ret); 168219820Sjeff goto out; 169219820Sjeff } 170219820Sjeffout: 171219820Sjeff return ret; 172219820Sjeff} 173219820Sjeff 174219820Sjeffstatic int post_recvs(struct cmatest_node *node) 175219820Sjeff{ 176219820Sjeff struct ibv_recv_wr recv_wr, *recv_failure; 177219820Sjeff struct ibv_sge sge; 178219820Sjeff int i, ret = 0; 179219820Sjeff 180219820Sjeff if (!message_count) 181219820Sjeff return 0; 182219820Sjeff 183219820Sjeff recv_wr.next = NULL; 184219820Sjeff recv_wr.sg_list = &sge; 185219820Sjeff recv_wr.num_sge = 1; 186219820Sjeff recv_wr.wr_id = (uintptr_t) node; 187219820Sjeff 188219820Sjeff sge.length = message_size + sizeof(struct ibv_grh); 189219820Sjeff sge.lkey = node->mr->lkey; 190219820Sjeff sge.addr = (uintptr_t) node->mem; 191219820Sjeff 192219820Sjeff for (i = 0; i < message_count && !ret; i++ ) { 193219820Sjeff ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure); 194219820Sjeff if (ret) { 195219820Sjeff printf("failed to post receives: %d\n", ret); 196219820Sjeff break; 197219820Sjeff } 198219820Sjeff } 199219820Sjeff return ret; 200219820Sjeff} 201219820Sjeff 202219820Sjeffstatic int post_sends(struct cmatest_node *node, int signal_flag) 203219820Sjeff{ 204219820Sjeff struct ibv_send_wr send_wr, *bad_send_wr; 205219820Sjeff struct ibv_sge sge; 206219820Sjeff int i, ret = 0; 207219820Sjeff 208219820Sjeff if (!node->connected || !message_count) 209219820Sjeff return 0; 210219820Sjeff 211219820Sjeff send_wr.next = NULL; 212219820Sjeff send_wr.sg_list = &sge; 213219820Sjeff send_wr.num_sge = 1; 214219820Sjeff send_wr.opcode = IBV_WR_SEND_WITH_IMM; 215219820Sjeff send_wr.send_flags = signal_flag; 216219820Sjeff send_wr.wr_id = (unsigned long)node; 217219820Sjeff send_wr.imm_data = htonl(node->cma_id->qp->qp_num); 218219820Sjeff 219219820Sjeff send_wr.wr.ud.ah = node->ah; 220219820Sjeff send_wr.wr.ud.remote_qpn = node->remote_qpn; 221219820Sjeff send_wr.wr.ud.remote_qkey = node->remote_qkey; 222219820Sjeff 223219820Sjeff sge.length = message_size; 224219820Sjeff sge.lkey = node->mr->lkey; 225219820Sjeff sge.addr = (uintptr_t) node->mem; 226219820Sjeff 227219820Sjeff for (i = 0; i < message_count && !ret; i++) { 228219820Sjeff ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr); 229219820Sjeff if (ret) 230219820Sjeff printf("failed to post sends: %d\n", ret); 231219820Sjeff } 232219820Sjeff return ret; 233219820Sjeff} 234219820Sjeff 235219820Sjeffstatic void connect_error(void) 236219820Sjeff{ 237219820Sjeff test.connects_left--; 238219820Sjeff} 239219820Sjeff 240219820Sjeffstatic int addr_handler(struct cmatest_node *node) 241219820Sjeff{ 242219820Sjeff int ret; 243219820Sjeff 244219820Sjeff if (set_tos) { 245219820Sjeff ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID, 246219820Sjeff RDMA_OPTION_ID_TOS, &tos, sizeof tos); 247219820Sjeff if (ret) 248219820Sjeff perror("udaddy: set TOS option failed"); 249219820Sjeff } 250219820Sjeff 251219820Sjeff ret = rdma_resolve_route(node->cma_id, 2000); 252219820Sjeff if (ret) { 253219820Sjeff perror("udaddy: resolve route failed"); 254219820Sjeff connect_error(); 255219820Sjeff } 256219820Sjeff return ret; 257219820Sjeff} 258219820Sjeff 259219820Sjeffstatic int route_handler(struct cmatest_node *node) 260219820Sjeff{ 261219820Sjeff struct rdma_conn_param conn_param; 262219820Sjeff int ret; 263219820Sjeff 264219820Sjeff ret = verify_test_params(node); 265219820Sjeff if (ret) 266219820Sjeff goto err; 267219820Sjeff 268219820Sjeff ret = init_node(node); 269219820Sjeff if (ret) 270219820Sjeff goto err; 271219820Sjeff 272219820Sjeff ret = post_recvs(node); 273219820Sjeff if (ret) 274219820Sjeff goto err; 275219820Sjeff 276219820Sjeff memset(&conn_param, 0, sizeof conn_param); 277219820Sjeff ret = rdma_connect(node->cma_id, &conn_param); 278219820Sjeff if (ret) { 279219820Sjeff perror("udaddy: failure connecting"); 280219820Sjeff goto err; 281219820Sjeff } 282219820Sjeff return 0; 283219820Sjefferr: 284219820Sjeff connect_error(); 285219820Sjeff return ret; 286219820Sjeff} 287219820Sjeff 288219820Sjeffstatic int connect_handler(struct rdma_cm_id *cma_id) 289219820Sjeff{ 290219820Sjeff struct cmatest_node *node; 291219820Sjeff struct rdma_conn_param conn_param; 292219820Sjeff int ret; 293219820Sjeff 294219820Sjeff if (test.conn_index == connections) { 295219820Sjeff ret = -ENOMEM; 296219820Sjeff goto err1; 297219820Sjeff } 298219820Sjeff node = &test.nodes[test.conn_index++]; 299219820Sjeff 300219820Sjeff node->cma_id = cma_id; 301219820Sjeff cma_id->context = node; 302219820Sjeff 303219820Sjeff ret = verify_test_params(node); 304219820Sjeff if (ret) 305219820Sjeff goto err2; 306219820Sjeff 307219820Sjeff ret = init_node(node); 308219820Sjeff if (ret) 309219820Sjeff goto err2; 310219820Sjeff 311219820Sjeff ret = post_recvs(node); 312219820Sjeff if (ret) 313219820Sjeff goto err2; 314219820Sjeff 315219820Sjeff memset(&conn_param, 0, sizeof conn_param); 316219820Sjeff conn_param.qp_num = node->cma_id->qp->qp_num; 317219820Sjeff ret = rdma_accept(node->cma_id, &conn_param); 318219820Sjeff if (ret) { 319219820Sjeff perror("udaddy: failure accepting"); 320219820Sjeff goto err2; 321219820Sjeff } 322219820Sjeff node->connected = 1; 323219820Sjeff test.connects_left--; 324219820Sjeff return 0; 325219820Sjeff 326219820Sjefferr2: 327219820Sjeff node->cma_id = NULL; 328219820Sjeff connect_error(); 329219820Sjefferr1: 330219820Sjeff printf("udaddy: failing connection request\n"); 331219820Sjeff rdma_reject(cma_id, NULL, 0); 332219820Sjeff return ret; 333219820Sjeff} 334219820Sjeff 335219820Sjeffstatic int resolved_handler(struct cmatest_node *node, 336219820Sjeff struct rdma_cm_event *event) 337219820Sjeff{ 338219820Sjeff node->remote_qpn = event->param.ud.qp_num; 339219820Sjeff node->remote_qkey = event->param.ud.qkey; 340219820Sjeff node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr); 341219820Sjeff if (!node->ah) { 342219820Sjeff printf("udaddy: failure creating address handle\n"); 343219820Sjeff goto err; 344219820Sjeff } 345219820Sjeff 346219820Sjeff node->connected = 1; 347219820Sjeff test.connects_left--; 348219820Sjeff return 0; 349219820Sjefferr: 350219820Sjeff connect_error(); 351219820Sjeff return -1; 352219820Sjeff} 353219820Sjeff 354219820Sjeffstatic int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 355219820Sjeff{ 356219820Sjeff int ret = 0; 357219820Sjeff 358219820Sjeff switch (event->event) { 359219820Sjeff case RDMA_CM_EVENT_ADDR_RESOLVED: 360219820Sjeff ret = addr_handler(cma_id->context); 361219820Sjeff break; 362219820Sjeff case RDMA_CM_EVENT_ROUTE_RESOLVED: 363219820Sjeff ret = route_handler(cma_id->context); 364219820Sjeff break; 365219820Sjeff case RDMA_CM_EVENT_CONNECT_REQUEST: 366219820Sjeff ret = connect_handler(cma_id); 367219820Sjeff break; 368219820Sjeff case RDMA_CM_EVENT_ESTABLISHED: 369219820Sjeff ret = resolved_handler(cma_id->context, event); 370219820Sjeff break; 371219820Sjeff case RDMA_CM_EVENT_ADDR_ERROR: 372219820Sjeff case RDMA_CM_EVENT_ROUTE_ERROR: 373219820Sjeff case RDMA_CM_EVENT_CONNECT_ERROR: 374219820Sjeff case RDMA_CM_EVENT_UNREACHABLE: 375219820Sjeff case RDMA_CM_EVENT_REJECTED: 376219820Sjeff printf("udaddy: event: %s, error: %d\n", 377219820Sjeff rdma_event_str(event->event), event->status); 378219820Sjeff connect_error(); 379219820Sjeff ret = event->status; 380219820Sjeff break; 381219820Sjeff case RDMA_CM_EVENT_DEVICE_REMOVAL: 382219820Sjeff /* Cleanup will occur after test completes. */ 383219820Sjeff break; 384219820Sjeff default: 385219820Sjeff break; 386219820Sjeff } 387219820Sjeff return ret; 388219820Sjeff} 389219820Sjeff 390219820Sjeffstatic void destroy_node(struct cmatest_node *node) 391219820Sjeff{ 392219820Sjeff if (!node->cma_id) 393219820Sjeff return; 394219820Sjeff 395219820Sjeff if (node->ah) 396219820Sjeff ibv_destroy_ah(node->ah); 397219820Sjeff 398219820Sjeff if (node->cma_id->qp) 399219820Sjeff rdma_destroy_qp(node->cma_id); 400219820Sjeff 401219820Sjeff if (node->cq) 402219820Sjeff ibv_destroy_cq(node->cq); 403219820Sjeff 404219820Sjeff if (node->mem) { 405219820Sjeff ibv_dereg_mr(node->mr); 406219820Sjeff free(node->mem); 407219820Sjeff } 408219820Sjeff 409219820Sjeff if (node->pd) 410219820Sjeff ibv_dealloc_pd(node->pd); 411219820Sjeff 412219820Sjeff /* Destroy the RDMA ID after all device resources */ 413219820Sjeff rdma_destroy_id(node->cma_id); 414219820Sjeff} 415219820Sjeff 416219820Sjeffstatic int alloc_nodes(void) 417219820Sjeff{ 418219820Sjeff int ret, i; 419219820Sjeff 420219820Sjeff test.nodes = malloc(sizeof *test.nodes * connections); 421219820Sjeff if (!test.nodes) { 422219820Sjeff printf("udaddy: unable to allocate memory for test nodes\n"); 423219820Sjeff return -ENOMEM; 424219820Sjeff } 425219820Sjeff memset(test.nodes, 0, sizeof *test.nodes * connections); 426219820Sjeff 427219820Sjeff for (i = 0; i < connections; i++) { 428219820Sjeff test.nodes[i].id = i; 429219820Sjeff if (dst_addr) { 430219820Sjeff ret = rdma_create_id(test.channel, 431219820Sjeff &test.nodes[i].cma_id, 432219820Sjeff &test.nodes[i], port_space); 433219820Sjeff if (ret) 434219820Sjeff goto err; 435219820Sjeff } 436219820Sjeff } 437219820Sjeff return 0; 438219820Sjefferr: 439219820Sjeff while (--i >= 0) 440219820Sjeff rdma_destroy_id(test.nodes[i].cma_id); 441219820Sjeff free(test.nodes); 442219820Sjeff return ret; 443219820Sjeff} 444219820Sjeff 445219820Sjeffstatic void destroy_nodes(void) 446219820Sjeff{ 447219820Sjeff int i; 448219820Sjeff 449219820Sjeff for (i = 0; i < connections; i++) 450219820Sjeff destroy_node(&test.nodes[i]); 451219820Sjeff free(test.nodes); 452219820Sjeff} 453219820Sjeff 454219820Sjeffstatic void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc) 455219820Sjeff{ 456219820Sjeff struct ibv_qp_attr attr; 457219820Sjeff struct ibv_qp_init_attr init_attr; 458219820Sjeff 459219820Sjeff node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem, 460219820Sjeff node->cma_id->port_num); 461219820Sjeff node->remote_qpn = ntohl(wc->imm_data); 462219820Sjeff 463219820Sjeff ibv_query_qp(node->cma_id->qp, &attr, IBV_QP_QKEY, &init_attr); 464219820Sjeff node->remote_qkey = attr.qkey; 465219820Sjeff} 466219820Sjeff 467219820Sjeffstatic int poll_cqs(void) 468219820Sjeff{ 469219820Sjeff struct ibv_wc wc[8]; 470219820Sjeff int done, i, ret; 471219820Sjeff 472219820Sjeff for (i = 0; i < connections; i++) { 473219820Sjeff if (!test.nodes[i].connected) 474219820Sjeff continue; 475219820Sjeff 476219820Sjeff for (done = 0; done < message_count; done += ret) { 477219820Sjeff ret = ibv_poll_cq(test.nodes[i].cq, 8, wc); 478219820Sjeff if (ret < 0) { 479219820Sjeff printf("udaddy: failed polling CQ: %d\n", ret); 480219820Sjeff return ret; 481219820Sjeff } 482219820Sjeff 483219820Sjeff if (ret && !test.nodes[i].ah) 484219820Sjeff create_reply_ah(&test.nodes[i], wc); 485219820Sjeff } 486219820Sjeff } 487219820Sjeff return 0; 488219820Sjeff} 489219820Sjeff 490219820Sjeffstatic int connect_events(void) 491219820Sjeff{ 492219820Sjeff struct rdma_cm_event *event; 493219820Sjeff int ret = 0; 494219820Sjeff 495219820Sjeff while (test.connects_left && !ret) { 496219820Sjeff ret = rdma_get_cm_event(test.channel, &event); 497219820Sjeff if (!ret) { 498219820Sjeff ret = cma_handler(event->id, event); 499219820Sjeff rdma_ack_cm_event(event); 500219820Sjeff } 501219820Sjeff } 502219820Sjeff return ret; 503219820Sjeff} 504219820Sjeff 505219820Sjeffstatic int get_addr(char *dst, struct sockaddr_in *addr) 506219820Sjeff{ 507219820Sjeff struct addrinfo *res; 508219820Sjeff int ret; 509219820Sjeff 510219820Sjeff ret = getaddrinfo(dst, NULL, NULL, &res); 511219820Sjeff if (ret) { 512219820Sjeff printf("getaddrinfo failed - invalid hostname or IP address\n"); 513219820Sjeff return ret; 514219820Sjeff } 515219820Sjeff 516219820Sjeff if (res->ai_family != PF_INET) { 517219820Sjeff ret = -1; 518219820Sjeff goto out; 519219820Sjeff } 520219820Sjeff 521219820Sjeff *addr = *(struct sockaddr_in *) res->ai_addr; 522219820Sjeffout: 523219820Sjeff freeaddrinfo(res); 524219820Sjeff return ret; 525219820Sjeff} 526219820Sjeff 527219820Sjeffstatic int run_server(void) 528219820Sjeff{ 529219820Sjeff struct rdma_cm_id *listen_id; 530219820Sjeff int i, ret; 531219820Sjeff 532219820Sjeff printf("udaddy: starting server\n"); 533219820Sjeff ret = rdma_create_id(test.channel, &listen_id, &test, port_space); 534219820Sjeff if (ret) { 535219820Sjeff perror("udaddy: listen request failed"); 536219820Sjeff return ret; 537219820Sjeff } 538219820Sjeff 539219820Sjeff if (src_addr) { 540219820Sjeff ret = get_addr(src_addr, &test.src_in); 541219820Sjeff if (ret) 542219820Sjeff goto out; 543219820Sjeff } else 544219820Sjeff test.src_in.sin_family = PF_INET; 545219820Sjeff 546219820Sjeff test.src_in.sin_port = port; 547219820Sjeff ret = rdma_bind_addr(listen_id, test.src_addr); 548219820Sjeff if (ret) { 549219820Sjeff perror("udaddy: bind address failed"); 550219820Sjeff return ret; 551219820Sjeff } 552219820Sjeff 553219820Sjeff ret = rdma_listen(listen_id, 0); 554219820Sjeff if (ret) { 555219820Sjeff perror("udaddy: failure trying to listen"); 556219820Sjeff goto out; 557219820Sjeff } 558219820Sjeff 559219820Sjeff connect_events(); 560219820Sjeff 561219820Sjeff if (message_count) { 562219820Sjeff printf("receiving data transfers\n"); 563219820Sjeff ret = poll_cqs(); 564219820Sjeff if (ret) 565219820Sjeff goto out; 566219820Sjeff 567219820Sjeff printf("sending replies\n"); 568219820Sjeff for (i = 0; i < connections; i++) { 569219820Sjeff ret = post_sends(&test.nodes[i], IBV_SEND_SIGNALED); 570219820Sjeff if (ret) 571219820Sjeff goto out; 572219820Sjeff } 573219820Sjeff 574219820Sjeff ret = poll_cqs(); 575219820Sjeff if (ret) 576219820Sjeff goto out; 577219820Sjeff printf("data transfers complete\n"); 578219820Sjeff } 579219820Sjeffout: 580219820Sjeff rdma_destroy_id(listen_id); 581219820Sjeff return ret; 582219820Sjeff} 583219820Sjeff 584219820Sjeffstatic int run_client(void) 585219820Sjeff{ 586219820Sjeff int i, ret; 587219820Sjeff 588219820Sjeff printf("udaddy: starting client\n"); 589219820Sjeff if (src_addr) { 590219820Sjeff ret = get_addr(src_addr, &test.src_in); 591219820Sjeff if (ret) 592219820Sjeff return ret; 593219820Sjeff } 594219820Sjeff 595219820Sjeff ret = get_addr(dst_addr, &test.dst_in); 596219820Sjeff if (ret) 597219820Sjeff return ret; 598219820Sjeff 599219820Sjeff test.dst_in.sin_port = port; 600219820Sjeff 601219820Sjeff printf("udaddy: connecting\n"); 602219820Sjeff for (i = 0; i < connections; i++) { 603219820Sjeff ret = rdma_resolve_addr(test.nodes[i].cma_id, 604219820Sjeff src_addr ? test.src_addr : NULL, 605219820Sjeff test.dst_addr, 2000); 606219820Sjeff if (ret) { 607219820Sjeff perror("udaddy: failure getting addr"); 608219820Sjeff connect_error(); 609219820Sjeff return ret; 610219820Sjeff } 611219820Sjeff } 612219820Sjeff 613219820Sjeff ret = connect_events(); 614219820Sjeff if (ret) 615219820Sjeff goto out; 616219820Sjeff 617219820Sjeff if (message_count) { 618219820Sjeff printf("initiating data transfers\n"); 619219820Sjeff for (i = 0; i < connections; i++) { 620219820Sjeff ret = post_sends(&test.nodes[i], 0); 621219820Sjeff if (ret) 622219820Sjeff goto out; 623219820Sjeff } 624219820Sjeff printf("receiving data transfers\n"); 625219820Sjeff ret = poll_cqs(); 626219820Sjeff if (ret) 627219820Sjeff goto out; 628219820Sjeff 629219820Sjeff printf("data transfers complete\n"); 630219820Sjeff } 631219820Sjeffout: 632219820Sjeff return ret; 633219820Sjeff} 634219820Sjeff 635219820Sjeffint main(int argc, char **argv) 636219820Sjeff{ 637219820Sjeff int op, ret; 638219820Sjeff 639219820Sjeff while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:")) != -1) { 640219820Sjeff switch (op) { 641219820Sjeff case 's': 642219820Sjeff dst_addr = optarg; 643219820Sjeff break; 644219820Sjeff case 'b': 645219820Sjeff src_addr = optarg; 646219820Sjeff break; 647219820Sjeff case 'c': 648219820Sjeff connections = atoi(optarg); 649219820Sjeff break; 650219820Sjeff case 'C': 651219820Sjeff message_count = atoi(optarg); 652219820Sjeff break; 653219820Sjeff case 'S': 654219820Sjeff message_size = atoi(optarg); 655219820Sjeff break; 656219820Sjeff case 't': 657219820Sjeff set_tos = 1; 658219820Sjeff tos = (uint8_t) atoi(optarg); 659219820Sjeff break; 660219820Sjeff case 'p': 661219820Sjeff port_space = strtol(optarg, NULL, 0); 662219820Sjeff break; 663219820Sjeff default: 664219820Sjeff printf("usage: %s\n", argv[0]); 665219820Sjeff printf("\t[-s server_address]\n"); 666219820Sjeff printf("\t[-b bind_address]\n"); 667219820Sjeff printf("\t[-c connections]\n"); 668219820Sjeff printf("\t[-C message_count]\n"); 669219820Sjeff printf("\t[-S message_size]\n"); 670219820Sjeff printf("\t[-t type_of_service]\n"); 671219820Sjeff printf("\t[-p port_space - %#x for UDP (default), " 672219820Sjeff "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB); 673219820Sjeff exit(1); 674219820Sjeff } 675219820Sjeff } 676219820Sjeff 677219820Sjeff test.dst_addr = (struct sockaddr *) &test.dst_in; 678219820Sjeff test.src_addr = (struct sockaddr *) &test.src_in; 679219820Sjeff test.connects_left = connections; 680219820Sjeff 681219820Sjeff test.channel = rdma_create_event_channel(); 682219820Sjeff if (!test.channel) { 683219820Sjeff perror("failed to create event channel"); 684219820Sjeff exit(1); 685219820Sjeff } 686219820Sjeff 687219820Sjeff if (alloc_nodes()) 688219820Sjeff exit(1); 689219820Sjeff 690219820Sjeff if (dst_addr) 691219820Sjeff ret = run_client(); 692219820Sjeff else 693219820Sjeff ret = run_server(); 694219820Sjeff 695219820Sjeff printf("test complete\n"); 696219820Sjeff destroy_nodes(); 697219820Sjeff rdma_destroy_event_channel(test.channel); 698219820Sjeff 699219820Sjeff printf("return status %d\n", ret); 700219820Sjeff return ret; 701219820Sjeff} 702