1219820Sjeff/* 2331769Shselasky * Copyright (c) 2005-2006,2011-2012 Intel Corporation. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff * $Id$ 33219820Sjeff */ 34219820Sjeff 35219820Sjeff#include <stdlib.h> 36219820Sjeff#include <string.h> 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <errno.h> 39219820Sjeff#include <sys/types.h> 40219820Sjeff#include <sys/socket.h> 41219820Sjeff#include <netdb.h> 42219820Sjeff#include <getopt.h> 43219820Sjeff 44219820Sjeff#include <rdma/rdma_cma.h> 45331769Shselasky#include "common.h" 46219820Sjeff 47219820Sjeffstruct cmatest_node { 48219820Sjeff int id; 49219820Sjeff struct rdma_cm_id *cma_id; 50219820Sjeff int connected; 51219820Sjeff struct ibv_pd *pd; 52219820Sjeff struct ibv_cq *cq[2]; 53219820Sjeff struct ibv_mr *mr; 54219820Sjeff void *mem; 55219820Sjeff}; 56219820Sjeff 57219820Sjeffenum CQ_INDEX { 58219820Sjeff SEND_CQ_INDEX, 59219820Sjeff RECV_CQ_INDEX 60219820Sjeff}; 61219820Sjeff 62219820Sjeffstruct cmatest { 63219820Sjeff struct rdma_event_channel *channel; 64219820Sjeff struct cmatest_node *nodes; 65219820Sjeff int conn_index; 66219820Sjeff int connects_left; 67219820Sjeff int disconnects_left; 68219820Sjeff 69331769Shselasky struct rdma_addrinfo *rai; 70219820Sjeff}; 71219820Sjeff 72219820Sjeffstatic struct cmatest test; 73219820Sjeffstatic int connections = 1; 74219820Sjeffstatic int message_size = 100; 75219820Sjeffstatic int message_count = 10; 76331769Shselaskystatic const char *port = "7471"; 77219820Sjeffstatic uint8_t set_tos = 0; 78219820Sjeffstatic uint8_t tos; 79219820Sjeffstatic uint8_t migrate = 0; 80219820Sjeffstatic char *dst_addr; 81219820Sjeffstatic char *src_addr; 82331769Shselaskystatic struct rdma_addrinfo hints; 83219820Sjeff 84219820Sjeffstatic int create_message(struct cmatest_node *node) 85219820Sjeff{ 86219820Sjeff if (!message_size) 87219820Sjeff message_count = 0; 88219820Sjeff 89219820Sjeff if (!message_count) 90219820Sjeff return 0; 91219820Sjeff 92219820Sjeff node->mem = malloc(message_size); 93219820Sjeff if (!node->mem) { 94219820Sjeff printf("failed message allocation\n"); 95219820Sjeff return -1; 96219820Sjeff } 97219820Sjeff node->mr = ibv_reg_mr(node->pd, node->mem, message_size, 98219820Sjeff IBV_ACCESS_LOCAL_WRITE); 99219820Sjeff if (!node->mr) { 100219820Sjeff printf("failed to reg MR\n"); 101219820Sjeff goto err; 102219820Sjeff } 103219820Sjeff return 0; 104219820Sjefferr: 105219820Sjeff free(node->mem); 106219820Sjeff return -1; 107219820Sjeff} 108219820Sjeff 109219820Sjeffstatic int init_node(struct cmatest_node *node) 110219820Sjeff{ 111219820Sjeff struct ibv_qp_init_attr init_qp_attr; 112219820Sjeff int cqe, ret; 113219820Sjeff 114219820Sjeff node->pd = ibv_alloc_pd(node->cma_id->verbs); 115219820Sjeff if (!node->pd) { 116219820Sjeff ret = -ENOMEM; 117219820Sjeff printf("cmatose: unable to allocate PD\n"); 118219820Sjeff goto out; 119219820Sjeff } 120219820Sjeff 121219820Sjeff cqe = message_count ? message_count : 1; 122331769Shselasky node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0); 123331769Shselasky node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0); 124219820Sjeff if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) { 125219820Sjeff ret = -ENOMEM; 126219820Sjeff printf("cmatose: unable to create CQ\n"); 127219820Sjeff goto out; 128219820Sjeff } 129219820Sjeff 130219820Sjeff memset(&init_qp_attr, 0, sizeof init_qp_attr); 131219820Sjeff init_qp_attr.cap.max_send_wr = cqe; 132219820Sjeff init_qp_attr.cap.max_recv_wr = cqe; 133219820Sjeff init_qp_attr.cap.max_send_sge = 1; 134219820Sjeff init_qp_attr.cap.max_recv_sge = 1; 135219820Sjeff init_qp_attr.qp_context = node; 136219820Sjeff init_qp_attr.sq_sig_all = 1; 137219820Sjeff init_qp_attr.qp_type = IBV_QPT_RC; 138219820Sjeff init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX]; 139219820Sjeff init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX]; 140219820Sjeff ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr); 141219820Sjeff if (ret) { 142219820Sjeff perror("cmatose: unable to create QP"); 143219820Sjeff goto out; 144219820Sjeff } 145219820Sjeff 146219820Sjeff ret = create_message(node); 147219820Sjeff if (ret) { 148219820Sjeff printf("cmatose: failed to create messages: %d\n", ret); 149219820Sjeff goto out; 150219820Sjeff } 151219820Sjeffout: 152219820Sjeff return ret; 153219820Sjeff} 154219820Sjeff 155219820Sjeffstatic int post_recvs(struct cmatest_node *node) 156219820Sjeff{ 157219820Sjeff struct ibv_recv_wr recv_wr, *recv_failure; 158219820Sjeff struct ibv_sge sge; 159219820Sjeff int i, ret = 0; 160219820Sjeff 161219820Sjeff if (!message_count) 162219820Sjeff return 0; 163219820Sjeff 164219820Sjeff recv_wr.next = NULL; 165219820Sjeff recv_wr.sg_list = &sge; 166219820Sjeff recv_wr.num_sge = 1; 167219820Sjeff recv_wr.wr_id = (uintptr_t) node; 168219820Sjeff 169219820Sjeff sge.length = message_size; 170219820Sjeff sge.lkey = node->mr->lkey; 171219820Sjeff sge.addr = (uintptr_t) node->mem; 172219820Sjeff 173219820Sjeff for (i = 0; i < message_count && !ret; i++ ) { 174219820Sjeff ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure); 175219820Sjeff if (ret) { 176219820Sjeff printf("failed to post receives: %d\n", ret); 177219820Sjeff break; 178219820Sjeff } 179219820Sjeff } 180219820Sjeff return ret; 181219820Sjeff} 182219820Sjeff 183219820Sjeffstatic int post_sends(struct cmatest_node *node) 184219820Sjeff{ 185219820Sjeff struct ibv_send_wr send_wr, *bad_send_wr; 186219820Sjeff struct ibv_sge sge; 187219820Sjeff int i, ret = 0; 188219820Sjeff 189219820Sjeff if (!node->connected || !message_count) 190219820Sjeff return 0; 191219820Sjeff 192219820Sjeff send_wr.next = NULL; 193219820Sjeff send_wr.sg_list = &sge; 194219820Sjeff send_wr.num_sge = 1; 195219820Sjeff send_wr.opcode = IBV_WR_SEND; 196219820Sjeff send_wr.send_flags = 0; 197219820Sjeff send_wr.wr_id = (unsigned long)node; 198219820Sjeff 199219820Sjeff sge.length = message_size; 200219820Sjeff sge.lkey = node->mr->lkey; 201219820Sjeff sge.addr = (uintptr_t) node->mem; 202219820Sjeff 203219820Sjeff for (i = 0; i < message_count && !ret; i++) { 204219820Sjeff ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr); 205219820Sjeff if (ret) 206219820Sjeff printf("failed to post sends: %d\n", ret); 207219820Sjeff } 208219820Sjeff return ret; 209219820Sjeff} 210219820Sjeff 211219820Sjeffstatic void connect_error(void) 212219820Sjeff{ 213219820Sjeff test.connects_left--; 214219820Sjeff} 215219820Sjeff 216219820Sjeffstatic int addr_handler(struct cmatest_node *node) 217219820Sjeff{ 218219820Sjeff int ret; 219219820Sjeff 220219820Sjeff if (set_tos) { 221219820Sjeff ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID, 222219820Sjeff RDMA_OPTION_ID_TOS, &tos, sizeof tos); 223219820Sjeff if (ret) 224219820Sjeff perror("cmatose: set TOS option failed"); 225219820Sjeff } 226219820Sjeff 227219820Sjeff ret = rdma_resolve_route(node->cma_id, 2000); 228219820Sjeff if (ret) { 229219820Sjeff perror("cmatose: resolve route failed"); 230219820Sjeff connect_error(); 231219820Sjeff } 232219820Sjeff return ret; 233219820Sjeff} 234219820Sjeff 235219820Sjeffstatic int route_handler(struct cmatest_node *node) 236219820Sjeff{ 237219820Sjeff struct rdma_conn_param conn_param; 238219820Sjeff int ret; 239219820Sjeff 240219820Sjeff ret = init_node(node); 241219820Sjeff if (ret) 242219820Sjeff goto err; 243219820Sjeff 244219820Sjeff ret = post_recvs(node); 245219820Sjeff if (ret) 246219820Sjeff goto err; 247219820Sjeff 248219820Sjeff memset(&conn_param, 0, sizeof conn_param); 249219820Sjeff conn_param.responder_resources = 1; 250219820Sjeff conn_param.initiator_depth = 1; 251219820Sjeff conn_param.retry_count = 5; 252331769Shselasky conn_param.private_data = test.rai->ai_connect; 253331769Shselasky conn_param.private_data_len = test.rai->ai_connect_len; 254219820Sjeff ret = rdma_connect(node->cma_id, &conn_param); 255219820Sjeff if (ret) { 256219820Sjeff perror("cmatose: failure connecting"); 257219820Sjeff goto err; 258219820Sjeff } 259219820Sjeff return 0; 260219820Sjefferr: 261219820Sjeff connect_error(); 262219820Sjeff return ret; 263219820Sjeff} 264219820Sjeff 265219820Sjeffstatic int connect_handler(struct rdma_cm_id *cma_id) 266219820Sjeff{ 267219820Sjeff struct cmatest_node *node; 268219820Sjeff int ret; 269219820Sjeff 270219820Sjeff if (test.conn_index == connections) { 271219820Sjeff ret = -ENOMEM; 272219820Sjeff goto err1; 273219820Sjeff } 274219820Sjeff node = &test.nodes[test.conn_index++]; 275219820Sjeff 276219820Sjeff node->cma_id = cma_id; 277219820Sjeff cma_id->context = node; 278219820Sjeff 279219820Sjeff ret = init_node(node); 280219820Sjeff if (ret) 281219820Sjeff goto err2; 282219820Sjeff 283219820Sjeff ret = post_recvs(node); 284219820Sjeff if (ret) 285219820Sjeff goto err2; 286219820Sjeff 287331769Shselasky ret = rdma_accept(node->cma_id, NULL); 288219820Sjeff if (ret) { 289219820Sjeff perror("cmatose: failure accepting"); 290219820Sjeff goto err2; 291219820Sjeff } 292219820Sjeff return 0; 293219820Sjeff 294219820Sjefferr2: 295219820Sjeff node->cma_id = NULL; 296219820Sjeff connect_error(); 297219820Sjefferr1: 298219820Sjeff printf("cmatose: failing connection request\n"); 299219820Sjeff rdma_reject(cma_id, NULL, 0); 300219820Sjeff return ret; 301219820Sjeff} 302219820Sjeff 303219820Sjeffstatic int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 304219820Sjeff{ 305219820Sjeff int ret = 0; 306219820Sjeff 307219820Sjeff switch (event->event) { 308219820Sjeff case RDMA_CM_EVENT_ADDR_RESOLVED: 309219820Sjeff ret = addr_handler(cma_id->context); 310219820Sjeff break; 311219820Sjeff case RDMA_CM_EVENT_ROUTE_RESOLVED: 312219820Sjeff ret = route_handler(cma_id->context); 313219820Sjeff break; 314219820Sjeff case RDMA_CM_EVENT_CONNECT_REQUEST: 315219820Sjeff ret = connect_handler(cma_id); 316219820Sjeff break; 317219820Sjeff case RDMA_CM_EVENT_ESTABLISHED: 318219820Sjeff ((struct cmatest_node *) cma_id->context)->connected = 1; 319219820Sjeff test.connects_left--; 320331769Shselasky test.disconnects_left++; 321219820Sjeff break; 322219820Sjeff case RDMA_CM_EVENT_ADDR_ERROR: 323219820Sjeff case RDMA_CM_EVENT_ROUTE_ERROR: 324219820Sjeff case RDMA_CM_EVENT_CONNECT_ERROR: 325219820Sjeff case RDMA_CM_EVENT_UNREACHABLE: 326219820Sjeff case RDMA_CM_EVENT_REJECTED: 327219820Sjeff printf("cmatose: event: %s, error: %d\n", 328219820Sjeff rdma_event_str(event->event), event->status); 329219820Sjeff connect_error(); 330331769Shselasky ret = event->status; 331219820Sjeff break; 332219820Sjeff case RDMA_CM_EVENT_DISCONNECTED: 333219820Sjeff rdma_disconnect(cma_id); 334219820Sjeff test.disconnects_left--; 335219820Sjeff break; 336219820Sjeff case RDMA_CM_EVENT_DEVICE_REMOVAL: 337219820Sjeff /* Cleanup will occur after test completes. */ 338219820Sjeff break; 339219820Sjeff default: 340219820Sjeff break; 341219820Sjeff } 342219820Sjeff return ret; 343219820Sjeff} 344219820Sjeff 345219820Sjeffstatic void destroy_node(struct cmatest_node *node) 346219820Sjeff{ 347219820Sjeff if (!node->cma_id) 348219820Sjeff return; 349219820Sjeff 350219820Sjeff if (node->cma_id->qp) 351219820Sjeff rdma_destroy_qp(node->cma_id); 352219820Sjeff 353219820Sjeff if (node->cq[SEND_CQ_INDEX]) 354219820Sjeff ibv_destroy_cq(node->cq[SEND_CQ_INDEX]); 355219820Sjeff 356219820Sjeff if (node->cq[RECV_CQ_INDEX]) 357219820Sjeff ibv_destroy_cq(node->cq[RECV_CQ_INDEX]); 358219820Sjeff 359219820Sjeff if (node->mem) { 360219820Sjeff ibv_dereg_mr(node->mr); 361219820Sjeff free(node->mem); 362219820Sjeff } 363219820Sjeff 364219820Sjeff if (node->pd) 365219820Sjeff ibv_dealloc_pd(node->pd); 366219820Sjeff 367219820Sjeff /* Destroy the RDMA ID after all device resources */ 368219820Sjeff rdma_destroy_id(node->cma_id); 369219820Sjeff} 370219820Sjeff 371219820Sjeffstatic int alloc_nodes(void) 372219820Sjeff{ 373219820Sjeff int ret, i; 374219820Sjeff 375219820Sjeff test.nodes = malloc(sizeof *test.nodes * connections); 376219820Sjeff if (!test.nodes) { 377219820Sjeff printf("cmatose: unable to allocate memory for test nodes\n"); 378219820Sjeff return -ENOMEM; 379219820Sjeff } 380219820Sjeff memset(test.nodes, 0, sizeof *test.nodes * connections); 381219820Sjeff 382219820Sjeff for (i = 0; i < connections; i++) { 383219820Sjeff test.nodes[i].id = i; 384219820Sjeff if (dst_addr) { 385219820Sjeff ret = rdma_create_id(test.channel, 386219820Sjeff &test.nodes[i].cma_id, 387331769Shselasky &test.nodes[i], hints.ai_port_space); 388219820Sjeff if (ret) 389219820Sjeff goto err; 390219820Sjeff } 391219820Sjeff } 392219820Sjeff return 0; 393219820Sjefferr: 394219820Sjeff while (--i >= 0) 395219820Sjeff rdma_destroy_id(test.nodes[i].cma_id); 396219820Sjeff free(test.nodes); 397219820Sjeff return ret; 398219820Sjeff} 399219820Sjeff 400219820Sjeffstatic void destroy_nodes(void) 401219820Sjeff{ 402219820Sjeff int i; 403219820Sjeff 404219820Sjeff for (i = 0; i < connections; i++) 405219820Sjeff destroy_node(&test.nodes[i]); 406219820Sjeff free(test.nodes); 407219820Sjeff} 408219820Sjeff 409219820Sjeffstatic int poll_cqs(enum CQ_INDEX index) 410219820Sjeff{ 411219820Sjeff struct ibv_wc wc[8]; 412219820Sjeff int done, i, ret; 413219820Sjeff 414219820Sjeff for (i = 0; i < connections; i++) { 415219820Sjeff if (!test.nodes[i].connected) 416219820Sjeff continue; 417219820Sjeff 418219820Sjeff for (done = 0; done < message_count; done += ret) { 419219820Sjeff ret = ibv_poll_cq(test.nodes[i].cq[index], 8, wc); 420219820Sjeff if (ret < 0) { 421219820Sjeff printf("cmatose: failed polling CQ: %d\n", ret); 422219820Sjeff return ret; 423219820Sjeff } 424219820Sjeff } 425219820Sjeff } 426219820Sjeff return 0; 427219820Sjeff} 428219820Sjeff 429219820Sjeffstatic int connect_events(void) 430219820Sjeff{ 431219820Sjeff struct rdma_cm_event *event; 432331769Shselasky int ret = 0; 433219820Sjeff 434331769Shselasky while (test.connects_left && !ret) { 435331769Shselasky ret = rdma_get_cm_event(test.channel, &event); 436331769Shselasky if (!ret) { 437331769Shselasky ret = cma_handler(event->id, event); 438219820Sjeff rdma_ack_cm_event(event); 439219820Sjeff } else { 440219820Sjeff perror("cmatose: failure in rdma_get_cm_event in connect events"); 441219820Sjeff ret = errno; 442219820Sjeff } 443219820Sjeff } 444219820Sjeff 445219820Sjeff return ret; 446219820Sjeff} 447219820Sjeff 448219820Sjeffstatic int disconnect_events(void) 449219820Sjeff{ 450219820Sjeff struct rdma_cm_event *event; 451331769Shselasky int ret = 0; 452219820Sjeff 453331769Shselasky while (test.disconnects_left && !ret) { 454331769Shselasky ret = rdma_get_cm_event(test.channel, &event); 455331769Shselasky if (!ret) { 456331769Shselasky ret = cma_handler(event->id, event); 457219820Sjeff rdma_ack_cm_event(event); 458219820Sjeff } else { 459219820Sjeff perror("cmatose: failure in rdma_get_cm_event in disconnect events"); 460219820Sjeff ret = errno; 461219820Sjeff } 462219820Sjeff } 463219820Sjeff 464219820Sjeff return ret; 465219820Sjeff} 466219820Sjeff 467219820Sjeffstatic int migrate_channel(struct rdma_cm_id *listen_id) 468219820Sjeff{ 469219820Sjeff struct rdma_event_channel *channel; 470219820Sjeff int i, ret; 471219820Sjeff 472219820Sjeff printf("migrating to new event channel\n"); 473219820Sjeff 474219820Sjeff channel = rdma_create_event_channel(); 475219820Sjeff if (!channel) { 476219820Sjeff perror("cmatose: failed to create event channel"); 477219820Sjeff return -1; 478219820Sjeff } 479219820Sjeff 480219820Sjeff ret = 0; 481219820Sjeff if (listen_id) 482219820Sjeff ret = rdma_migrate_id(listen_id, channel); 483219820Sjeff 484219820Sjeff for (i = 0; i < connections && !ret; i++) 485219820Sjeff ret = rdma_migrate_id(test.nodes[i].cma_id, channel); 486219820Sjeff 487219820Sjeff if (!ret) { 488219820Sjeff rdma_destroy_event_channel(test.channel); 489219820Sjeff test.channel = channel; 490219820Sjeff } else 491219820Sjeff perror("cmatose: failure migrating to channel"); 492219820Sjeff 493219820Sjeff return ret; 494219820Sjeff} 495219820Sjeff 496219820Sjeffstatic int run_server(void) 497219820Sjeff{ 498219820Sjeff struct rdma_cm_id *listen_id; 499219820Sjeff int i, ret; 500219820Sjeff 501219820Sjeff printf("cmatose: starting server\n"); 502331769Shselasky ret = rdma_create_id(test.channel, &listen_id, &test, hints.ai_port_space); 503219820Sjeff if (ret) { 504219820Sjeff perror("cmatose: listen request failed"); 505219820Sjeff return ret; 506219820Sjeff } 507219820Sjeff 508331769Shselasky ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai); 509331769Shselasky if (ret) { 510331769Shselasky printf("cmatose: getrdmaaddr error: %s\n", gai_strerror(ret)); 511331769Shselasky goto out; 512219820Sjeff } 513219820Sjeff 514331769Shselasky ret = rdma_bind_addr(listen_id, test.rai->ai_src_addr); 515219820Sjeff if (ret) { 516219820Sjeff perror("cmatose: bind address failed"); 517219820Sjeff goto out; 518219820Sjeff } 519219820Sjeff 520219820Sjeff ret = rdma_listen(listen_id, 0); 521219820Sjeff if (ret) { 522219820Sjeff perror("cmatose: failure trying to listen"); 523219820Sjeff goto out; 524219820Sjeff } 525219820Sjeff 526219820Sjeff ret = connect_events(); 527219820Sjeff if (ret) 528219820Sjeff goto out; 529219820Sjeff 530219820Sjeff if (message_count) { 531219820Sjeff printf("initiating data transfers\n"); 532219820Sjeff for (i = 0; i < connections; i++) { 533219820Sjeff ret = post_sends(&test.nodes[i]); 534219820Sjeff if (ret) 535219820Sjeff goto out; 536219820Sjeff } 537219820Sjeff 538219820Sjeff printf("completing sends\n"); 539219820Sjeff ret = poll_cqs(SEND_CQ_INDEX); 540219820Sjeff if (ret) 541219820Sjeff goto out; 542219820Sjeff 543219820Sjeff printf("receiving data transfers\n"); 544219820Sjeff ret = poll_cqs(RECV_CQ_INDEX); 545219820Sjeff if (ret) 546219820Sjeff goto out; 547219820Sjeff printf("data transfers complete\n"); 548219820Sjeff 549219820Sjeff } 550219820Sjeff 551219820Sjeff if (migrate) { 552219820Sjeff ret = migrate_channel(listen_id); 553219820Sjeff if (ret) 554219820Sjeff goto out; 555219820Sjeff } 556219820Sjeff 557219820Sjeff printf("cmatose: disconnecting\n"); 558219820Sjeff for (i = 0; i < connections; i++) { 559219820Sjeff if (!test.nodes[i].connected) 560219820Sjeff continue; 561219820Sjeff 562219820Sjeff test.nodes[i].connected = 0; 563219820Sjeff rdma_disconnect(test.nodes[i].cma_id); 564219820Sjeff } 565219820Sjeff 566219820Sjeff ret = disconnect_events(); 567219820Sjeff 568219820Sjeff printf("disconnected\n"); 569219820Sjeff 570219820Sjeffout: 571219820Sjeff rdma_destroy_id(listen_id); 572219820Sjeff return ret; 573219820Sjeff} 574219820Sjeff 575219820Sjeffstatic int run_client(void) 576219820Sjeff{ 577219820Sjeff int i, ret, ret2; 578219820Sjeff 579219820Sjeff printf("cmatose: starting client\n"); 580219820Sjeff 581331769Shselasky ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai); 582331769Shselasky if (ret) { 583331769Shselasky printf("cmatose: getaddrinfo error: %s\n", gai_strerror(ret)); 584219820Sjeff return ret; 585331769Shselasky } 586219820Sjeff 587219820Sjeff printf("cmatose: connecting\n"); 588219820Sjeff for (i = 0; i < connections; i++) { 589331769Shselasky ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr, 590331769Shselasky test.rai->ai_dst_addr, 2000); 591219820Sjeff if (ret) { 592219820Sjeff perror("cmatose: failure getting addr"); 593219820Sjeff connect_error(); 594219820Sjeff return ret; 595219820Sjeff } 596219820Sjeff } 597219820Sjeff 598219820Sjeff ret = connect_events(); 599219820Sjeff if (ret) 600219820Sjeff goto disc; 601219820Sjeff 602219820Sjeff if (message_count) { 603219820Sjeff printf("receiving data transfers\n"); 604219820Sjeff ret = poll_cqs(RECV_CQ_INDEX); 605219820Sjeff if (ret) 606219820Sjeff goto disc; 607219820Sjeff 608219820Sjeff printf("sending replies\n"); 609219820Sjeff for (i = 0; i < connections; i++) { 610219820Sjeff ret = post_sends(&test.nodes[i]); 611219820Sjeff if (ret) 612219820Sjeff goto disc; 613219820Sjeff } 614219820Sjeff 615219820Sjeff printf("data transfers complete\n"); 616219820Sjeff } 617219820Sjeff 618219820Sjeff ret = 0; 619219820Sjeff 620219820Sjeff if (migrate) { 621219820Sjeff ret = migrate_channel(NULL); 622219820Sjeff if (ret) 623219820Sjeff goto out; 624219820Sjeff } 625219820Sjeffdisc: 626219820Sjeff ret2 = disconnect_events(); 627219820Sjeff if (ret2) 628219820Sjeff ret = ret2; 629219820Sjeffout: 630219820Sjeff return ret; 631219820Sjeff} 632219820Sjeff 633219820Sjeffint main(int argc, char **argv) 634219820Sjeff{ 635219820Sjeff int op, ret; 636219820Sjeff 637331769Shselasky hints.ai_port_space = RDMA_PS_TCP; 638331769Shselasky while ((op = getopt(argc, argv, "s:b:f:P:c:C:S:t:p:m")) != -1) { 639219820Sjeff switch (op) { 640219820Sjeff case 's': 641219820Sjeff dst_addr = optarg; 642219820Sjeff break; 643219820Sjeff case 'b': 644219820Sjeff src_addr = optarg; 645219820Sjeff break; 646331769Shselasky case 'f': 647331769Shselasky if (!strncasecmp("ip", optarg, 2)) { 648331769Shselasky hints.ai_flags = RAI_NUMERICHOST; 649331769Shselasky } else if (!strncasecmp("gid", optarg, 3)) { 650331769Shselasky hints.ai_flags = RAI_NUMERICHOST | RAI_FAMILY; 651331769Shselasky hints.ai_family = AF_IB; 652331769Shselasky } else if (strncasecmp("name", optarg, 4)) { 653331769Shselasky fprintf(stderr, "Warning: unknown address format\n"); 654331769Shselasky } 655331769Shselasky break; 656331769Shselasky case 'P': 657331769Shselasky if (!strncasecmp("ib", optarg, 2)) { 658331769Shselasky hints.ai_port_space = RDMA_PS_IB; 659331769Shselasky } else if (strncasecmp("tcp", optarg, 3)) { 660331769Shselasky fprintf(stderr, "Warning: unknown port space format\n"); 661331769Shselasky } 662331769Shselasky break; 663219820Sjeff case 'c': 664219820Sjeff connections = atoi(optarg); 665219820Sjeff break; 666219820Sjeff case 'C': 667219820Sjeff message_count = atoi(optarg); 668219820Sjeff break; 669219820Sjeff case 'S': 670219820Sjeff message_size = atoi(optarg); 671219820Sjeff break; 672219820Sjeff case 't': 673219820Sjeff set_tos = 1; 674331769Shselasky tos = (uint8_t) strtoul(optarg, NULL, 0); 675219820Sjeff break; 676219820Sjeff case 'p': 677331769Shselasky port = optarg; 678219820Sjeff break; 679219820Sjeff case 'm': 680219820Sjeff migrate = 1; 681219820Sjeff break; 682219820Sjeff default: 683219820Sjeff printf("usage: %s\n", argv[0]); 684219820Sjeff printf("\t[-s server_address]\n"); 685219820Sjeff printf("\t[-b bind_address]\n"); 686331769Shselasky printf("\t[-f address_format]\n"); 687331769Shselasky printf("\t name, ip, ipv6, or gid\n"); 688331769Shselasky printf("\t[-P port_space]\n"); 689331769Shselasky printf("\t tcp or ib\n"); 690219820Sjeff printf("\t[-c connections]\n"); 691219820Sjeff printf("\t[-C message_count]\n"); 692219820Sjeff printf("\t[-S message_size]\n"); 693219820Sjeff printf("\t[-t type_of_service]\n"); 694219820Sjeff printf("\t[-p port_number]\n"); 695219820Sjeff printf("\t[-m(igrate)]\n"); 696219820Sjeff exit(1); 697219820Sjeff } 698219820Sjeff } 699219820Sjeff 700219820Sjeff test.connects_left = connections; 701219820Sjeff 702219820Sjeff test.channel = rdma_create_event_channel(); 703219820Sjeff if (!test.channel) { 704219820Sjeff printf("failed to create event channel\n"); 705219820Sjeff exit(1); 706219820Sjeff } 707219820Sjeff 708219820Sjeff if (alloc_nodes()) 709219820Sjeff exit(1); 710219820Sjeff 711331769Shselasky if (dst_addr) { 712219820Sjeff ret = run_client(); 713331769Shselasky } else { 714331769Shselasky hints.ai_flags |= RAI_PASSIVE; 715219820Sjeff ret = run_server(); 716331769Shselasky } 717219820Sjeff 718219820Sjeff printf("test complete\n"); 719219820Sjeff destroy_nodes(); 720219820Sjeff rdma_destroy_event_channel(test.channel); 721331769Shselasky if (test.rai) 722331769Shselasky rdma_freeaddrinfo(test.rai); 723219820Sjeff 724219820Sjeff printf("return status %d\n", ret); 725219820Sjeff return ret; 726219820Sjeff} 727