1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff * $Id$ 33219820Sjeff */ 34219820Sjeff 35219820Sjeff#include <stdlib.h> 36219820Sjeff#include <string.h> 37219820Sjeff#include <stdio.h> 38219820Sjeff#include <errno.h> 39219820Sjeff#include <sys/types.h> 40219820Sjeff#include <netinet/in.h> 41219820Sjeff#include <sys/socket.h> 42219820Sjeff#include <netdb.h> 43219820Sjeff#include <byteswap.h> 44219820Sjeff#include <getopt.h> 45219820Sjeff 46219820Sjeff#include <rdma/rdma_cma.h> 47219820Sjeff 48219820Sjeff#if __BYTE_ORDER == __BIG_ENDIAN 49219820Sjeffstatic inline uint64_t cpu_to_be64(uint64_t x) { return x; } 50219820Sjeffstatic inline uint32_t cpu_to_be32(uint32_t x) { return x; } 51219820Sjeff#else 52219820Sjeffstatic inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); } 53219820Sjeffstatic inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); } 54219820Sjeff#endif 55219820Sjeff 56219820Sjeffstruct cmatest_node { 57219820Sjeff int id; 58219820Sjeff struct rdma_cm_id *cma_id; 59219820Sjeff int connected; 60219820Sjeff struct ibv_pd *pd; 61219820Sjeff struct ibv_cq *cq[2]; 62219820Sjeff struct ibv_mr *mr; 63219820Sjeff void *mem; 64219820Sjeff}; 65219820Sjeff 66219820Sjeffenum CQ_INDEX { 67219820Sjeff SEND_CQ_INDEX, 68219820Sjeff RECV_CQ_INDEX 69219820Sjeff}; 70219820Sjeff 71219820Sjeffstruct cmatest { 72219820Sjeff struct rdma_event_channel *channel; 73219820Sjeff struct cmatest_node *nodes; 74219820Sjeff int conn_index; 75219820Sjeff int connects_left; 76219820Sjeff int disconnects_left; 77219820Sjeff 78219820Sjeff struct rdma_addr addr; 79219820Sjeff}; 80219820Sjeff 81219820Sjeffstatic struct cmatest test; 82219820Sjeffstatic int connections = 1; 83219820Sjeffstatic int message_size = 100; 84219820Sjeffstatic int message_count = 10; 85219820Sjeffstatic uint16_t port = 7471; 86219820Sjeffstatic uint8_t set_tos = 0; 87219820Sjeffstatic uint8_t tos; 88219820Sjeffstatic uint8_t migrate = 0; 89219820Sjeffstatic char *dst_addr; 90219820Sjeffstatic char *src_addr; 91219820Sjeff 92219820Sjeffstatic int create_message(struct cmatest_node *node) 93219820Sjeff{ 94219820Sjeff if (!message_size) 95219820Sjeff message_count = 0; 96219820Sjeff 97219820Sjeff if (!message_count) 98219820Sjeff return 0; 99219820Sjeff 100219820Sjeff node->mem = malloc(message_size); 101219820Sjeff if (!node->mem) { 102219820Sjeff printf("failed message allocation\n"); 103219820Sjeff return -1; 104219820Sjeff } 105219820Sjeff node->mr = ibv_reg_mr(node->pd, node->mem, message_size, 106219820Sjeff IBV_ACCESS_LOCAL_WRITE); 107219820Sjeff if (!node->mr) { 108219820Sjeff printf("failed to reg MR\n"); 109219820Sjeff goto err; 110219820Sjeff } 111219820Sjeff return 0; 112219820Sjefferr: 113219820Sjeff free(node->mem); 114219820Sjeff return -1; 115219820Sjeff} 116219820Sjeff 117219820Sjeffstatic int init_node(struct cmatest_node *node) 118219820Sjeff{ 119219820Sjeff struct ibv_qp_init_attr init_qp_attr; 120219820Sjeff int cqe, ret; 121219820Sjeff 122219820Sjeff node->pd = ibv_alloc_pd(node->cma_id->verbs); 123219820Sjeff if (!node->pd) { 124219820Sjeff ret = -ENOMEM; 125219820Sjeff printf("cmatose: unable to allocate PD\n"); 126219820Sjeff goto out; 127219820Sjeff } 128219820Sjeff 129219820Sjeff cqe = message_count ? message_count : 1; 130219820Sjeff node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0); 131219820Sjeff node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0); 132219820Sjeff if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) { 133219820Sjeff ret = -ENOMEM; 134219820Sjeff printf("cmatose: unable to create CQ\n"); 135219820Sjeff goto out; 136219820Sjeff } 137219820Sjeff 138219820Sjeff memset(&init_qp_attr, 0, sizeof init_qp_attr); 139219820Sjeff init_qp_attr.cap.max_send_wr = cqe; 140219820Sjeff init_qp_attr.cap.max_recv_wr = cqe; 141219820Sjeff init_qp_attr.cap.max_send_sge = 1; 142219820Sjeff init_qp_attr.cap.max_recv_sge = 1; 143219820Sjeff init_qp_attr.qp_context = node; 144219820Sjeff init_qp_attr.sq_sig_all = 1; 145219820Sjeff init_qp_attr.qp_type = IBV_QPT_RC; 146219820Sjeff init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX]; 147219820Sjeff init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX]; 148219820Sjeff ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr); 149219820Sjeff if (ret) { 150219820Sjeff perror("cmatose: unable to create QP"); 151219820Sjeff goto out; 152219820Sjeff } 153219820Sjeff 154219820Sjeff ret = create_message(node); 155219820Sjeff if (ret) { 156219820Sjeff printf("cmatose: failed to create messages: %d\n", ret); 157219820Sjeff goto out; 158219820Sjeff } 159219820Sjeffout: 160219820Sjeff return ret; 161219820Sjeff} 162219820Sjeff 163219820Sjeffstatic int post_recvs(struct cmatest_node *node) 164219820Sjeff{ 165219820Sjeff struct ibv_recv_wr recv_wr, *recv_failure; 166219820Sjeff struct ibv_sge sge; 167219820Sjeff int i, ret = 0; 168219820Sjeff 169219820Sjeff if (!message_count) 170219820Sjeff return 0; 171219820Sjeff 172219820Sjeff recv_wr.next = NULL; 173219820Sjeff recv_wr.sg_list = &sge; 174219820Sjeff recv_wr.num_sge = 1; 175219820Sjeff recv_wr.wr_id = (uintptr_t) node; 176219820Sjeff 177219820Sjeff sge.length = message_size; 178219820Sjeff sge.lkey = node->mr->lkey; 179219820Sjeff sge.addr = (uintptr_t) node->mem; 180219820Sjeff 181219820Sjeff for (i = 0; i < message_count && !ret; i++ ) { 182219820Sjeff ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure); 183219820Sjeff if (ret) { 184219820Sjeff printf("failed to post receives: %d\n", ret); 185219820Sjeff break; 186219820Sjeff } 187219820Sjeff } 188219820Sjeff return ret; 189219820Sjeff} 190219820Sjeff 191219820Sjeffstatic int post_sends(struct cmatest_node *node) 192219820Sjeff{ 193219820Sjeff struct ibv_send_wr send_wr, *bad_send_wr; 194219820Sjeff struct ibv_sge sge; 195219820Sjeff int i, ret = 0; 196219820Sjeff 197219820Sjeff if (!node->connected || !message_count) 198219820Sjeff return 0; 199219820Sjeff 200219820Sjeff send_wr.next = NULL; 201219820Sjeff send_wr.sg_list = &sge; 202219820Sjeff send_wr.num_sge = 1; 203219820Sjeff send_wr.opcode = IBV_WR_SEND; 204219820Sjeff send_wr.send_flags = 0; 205219820Sjeff send_wr.wr_id = (unsigned long)node; 206219820Sjeff 207219820Sjeff sge.length = message_size; 208219820Sjeff sge.lkey = node->mr->lkey; 209219820Sjeff sge.addr = (uintptr_t) node->mem; 210219820Sjeff 211219820Sjeff for (i = 0; i < message_count && !ret; i++) { 212219820Sjeff ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr); 213219820Sjeff if (ret) 214219820Sjeff printf("failed to post sends: %d\n", ret); 215219820Sjeff } 216219820Sjeff return ret; 217219820Sjeff} 218219820Sjeff 219219820Sjeffstatic void connect_error(void) 220219820Sjeff{ 221219820Sjeff test.disconnects_left--; 222219820Sjeff test.connects_left--; 223219820Sjeff} 224219820Sjeff 225219820Sjeffstatic int addr_handler(struct cmatest_node *node) 226219820Sjeff{ 227219820Sjeff int ret; 228219820Sjeff 229219820Sjeff if (set_tos) { 230219820Sjeff ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID, 231219820Sjeff RDMA_OPTION_ID_TOS, &tos, sizeof tos); 232219820Sjeff if (ret) 233219820Sjeff perror("cmatose: set TOS option failed"); 234219820Sjeff } 235219820Sjeff 236219820Sjeff ret = rdma_resolve_route(node->cma_id, 2000); 237219820Sjeff if (ret) { 238219820Sjeff perror("cmatose: resolve route failed"); 239219820Sjeff connect_error(); 240219820Sjeff } 241219820Sjeff return ret; 242219820Sjeff} 243219820Sjeff 244219820Sjeffstatic int route_handler(struct cmatest_node *node) 245219820Sjeff{ 246219820Sjeff struct rdma_conn_param conn_param; 247219820Sjeff int ret; 248219820Sjeff 249219820Sjeff ret = init_node(node); 250219820Sjeff if (ret) 251219820Sjeff goto err; 252219820Sjeff 253219820Sjeff ret = post_recvs(node); 254219820Sjeff if (ret) 255219820Sjeff goto err; 256219820Sjeff 257219820Sjeff memset(&conn_param, 0, sizeof conn_param); 258219820Sjeff conn_param.responder_resources = 1; 259219820Sjeff conn_param.initiator_depth = 1; 260219820Sjeff conn_param.retry_count = 5; 261219820Sjeff ret = rdma_connect(node->cma_id, &conn_param); 262219820Sjeff if (ret) { 263219820Sjeff perror("cmatose: failure connecting"); 264219820Sjeff goto err; 265219820Sjeff } 266219820Sjeff return 0; 267219820Sjefferr: 268219820Sjeff connect_error(); 269219820Sjeff return ret; 270219820Sjeff} 271219820Sjeff 272219820Sjeffstatic int connect_handler(struct rdma_cm_id *cma_id) 273219820Sjeff{ 274219820Sjeff struct cmatest_node *node; 275219820Sjeff struct rdma_conn_param conn_param; 276219820Sjeff int ret; 277219820Sjeff 278219820Sjeff if (test.conn_index == connections) { 279219820Sjeff ret = -ENOMEM; 280219820Sjeff goto err1; 281219820Sjeff } 282219820Sjeff node = &test.nodes[test.conn_index++]; 283219820Sjeff 284219820Sjeff node->cma_id = cma_id; 285219820Sjeff cma_id->context = node; 286219820Sjeff 287219820Sjeff ret = init_node(node); 288219820Sjeff if (ret) 289219820Sjeff goto err2; 290219820Sjeff 291219820Sjeff ret = post_recvs(node); 292219820Sjeff if (ret) 293219820Sjeff goto err2; 294219820Sjeff 295219820Sjeff memset(&conn_param, 0, sizeof conn_param); 296219820Sjeff conn_param.responder_resources = 1; 297219820Sjeff conn_param.initiator_depth = 1; 298219820Sjeff ret = rdma_accept(node->cma_id, &conn_param); 299219820Sjeff if (ret) { 300219820Sjeff perror("cmatose: failure accepting"); 301219820Sjeff goto err2; 302219820Sjeff } 303219820Sjeff return 0; 304219820Sjeff 305219820Sjefferr2: 306219820Sjeff node->cma_id = NULL; 307219820Sjeff connect_error(); 308219820Sjefferr1: 309219820Sjeff printf("cmatose: failing connection request\n"); 310219820Sjeff rdma_reject(cma_id, NULL, 0); 311219820Sjeff return ret; 312219820Sjeff} 313219820Sjeff 314219820Sjeffstatic int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 315219820Sjeff{ 316219820Sjeff int ret = 0; 317219820Sjeff 318219820Sjeff switch (event->event) { 319219820Sjeff case RDMA_CM_EVENT_ADDR_RESOLVED: 320219820Sjeff ret = addr_handler(cma_id->context); 321219820Sjeff break; 322219820Sjeff case RDMA_CM_EVENT_ROUTE_RESOLVED: 323219820Sjeff ret = route_handler(cma_id->context); 324219820Sjeff break; 325219820Sjeff case RDMA_CM_EVENT_CONNECT_REQUEST: 326219820Sjeff ret = connect_handler(cma_id); 327219820Sjeff break; 328219820Sjeff case RDMA_CM_EVENT_ESTABLISHED: 329219820Sjeff ((struct cmatest_node *) cma_id->context)->connected = 1; 330219820Sjeff test.connects_left--; 331219820Sjeff break; 332219820Sjeff case RDMA_CM_EVENT_ADDR_ERROR: 333219820Sjeff case RDMA_CM_EVENT_ROUTE_ERROR: 334219820Sjeff case RDMA_CM_EVENT_CONNECT_ERROR: 335219820Sjeff case RDMA_CM_EVENT_UNREACHABLE: 336219820Sjeff case RDMA_CM_EVENT_REJECTED: 337219820Sjeff printf("cmatose: event: %s, error: %d\n", 338219820Sjeff rdma_event_str(event->event), event->status); 339219820Sjeff connect_error(); 340219820Sjeff break; 341219820Sjeff case RDMA_CM_EVENT_DISCONNECTED: 342219820Sjeff rdma_disconnect(cma_id); 343219820Sjeff test.disconnects_left--; 344219820Sjeff break; 345219820Sjeff case RDMA_CM_EVENT_DEVICE_REMOVAL: 346219820Sjeff /* Cleanup will occur after test completes. */ 347219820Sjeff break; 348219820Sjeff default: 349219820Sjeff break; 350219820Sjeff } 351219820Sjeff return ret; 352219820Sjeff} 353219820Sjeff 354219820Sjeffstatic void destroy_node(struct cmatest_node *node) 355219820Sjeff{ 356219820Sjeff if (!node->cma_id) 357219820Sjeff return; 358219820Sjeff 359219820Sjeff if (node->cma_id->qp) 360219820Sjeff rdma_destroy_qp(node->cma_id); 361219820Sjeff 362219820Sjeff if (node->cq[SEND_CQ_INDEX]) 363219820Sjeff ibv_destroy_cq(node->cq[SEND_CQ_INDEX]); 364219820Sjeff 365219820Sjeff if (node->cq[RECV_CQ_INDEX]) 366219820Sjeff ibv_destroy_cq(node->cq[RECV_CQ_INDEX]); 367219820Sjeff 368219820Sjeff if (node->mem) { 369219820Sjeff ibv_dereg_mr(node->mr); 370219820Sjeff free(node->mem); 371219820Sjeff } 372219820Sjeff 373219820Sjeff if (node->pd) 374219820Sjeff ibv_dealloc_pd(node->pd); 375219820Sjeff 376219820Sjeff /* Destroy the RDMA ID after all device resources */ 377219820Sjeff rdma_destroy_id(node->cma_id); 378219820Sjeff} 379219820Sjeff 380219820Sjeffstatic int alloc_nodes(void) 381219820Sjeff{ 382219820Sjeff int ret, i; 383219820Sjeff 384219820Sjeff test.nodes = malloc(sizeof *test.nodes * connections); 385219820Sjeff if (!test.nodes) { 386219820Sjeff printf("cmatose: unable to allocate memory for test nodes\n"); 387219820Sjeff return -ENOMEM; 388219820Sjeff } 389219820Sjeff memset(test.nodes, 0, sizeof *test.nodes * connections); 390219820Sjeff 391219820Sjeff for (i = 0; i < connections; i++) { 392219820Sjeff test.nodes[i].id = i; 393219820Sjeff if (dst_addr) { 394219820Sjeff ret = rdma_create_id(test.channel, 395219820Sjeff &test.nodes[i].cma_id, 396219820Sjeff &test.nodes[i], RDMA_PS_TCP); 397219820Sjeff if (ret) 398219820Sjeff goto err; 399219820Sjeff } 400219820Sjeff } 401219820Sjeff return 0; 402219820Sjefferr: 403219820Sjeff while (--i >= 0) 404219820Sjeff rdma_destroy_id(test.nodes[i].cma_id); 405219820Sjeff free(test.nodes); 406219820Sjeff return ret; 407219820Sjeff} 408219820Sjeff 409219820Sjeffstatic void destroy_nodes(void) 410219820Sjeff{ 411219820Sjeff int i; 412219820Sjeff 413219820Sjeff for (i = 0; i < connections; i++) 414219820Sjeff destroy_node(&test.nodes[i]); 415219820Sjeff free(test.nodes); 416219820Sjeff} 417219820Sjeff 418219820Sjeffstatic int poll_cqs(enum CQ_INDEX index) 419219820Sjeff{ 420219820Sjeff struct ibv_wc wc[8]; 421219820Sjeff int done, i, ret; 422219820Sjeff 423219820Sjeff for (i = 0; i < connections; i++) { 424219820Sjeff if (!test.nodes[i].connected) 425219820Sjeff continue; 426219820Sjeff 427219820Sjeff for (done = 0; done < message_count; done += ret) { 428219820Sjeff ret = ibv_poll_cq(test.nodes[i].cq[index], 8, wc); 429219820Sjeff if (ret < 0) { 430219820Sjeff printf("cmatose: failed polling CQ: %d\n", ret); 431219820Sjeff return ret; 432219820Sjeff } 433219820Sjeff } 434219820Sjeff } 435219820Sjeff return 0; 436219820Sjeff} 437219820Sjeff 438219820Sjeffstatic int connect_events(void) 439219820Sjeff{ 440219820Sjeff struct rdma_cm_event *event; 441219820Sjeff int err = 0, ret = 0; 442219820Sjeff 443219820Sjeff while (test.connects_left && !err) { 444219820Sjeff err = rdma_get_cm_event(test.channel, &event); 445219820Sjeff if (!err) { 446219820Sjeff cma_handler(event->id, event); 447219820Sjeff rdma_ack_cm_event(event); 448219820Sjeff } else { 449219820Sjeff perror("cmatose: failure in rdma_get_cm_event in connect events"); 450219820Sjeff ret = errno; 451219820Sjeff } 452219820Sjeff } 453219820Sjeff 454219820Sjeff return ret; 455219820Sjeff} 456219820Sjeff 457219820Sjeffstatic int disconnect_events(void) 458219820Sjeff{ 459219820Sjeff struct rdma_cm_event *event; 460219820Sjeff int err = 0, ret = 0; 461219820Sjeff 462219820Sjeff while (test.disconnects_left && !err) { 463219820Sjeff err = rdma_get_cm_event(test.channel, &event); 464219820Sjeff if (!err) { 465219820Sjeff cma_handler(event->id, event); 466219820Sjeff rdma_ack_cm_event(event); 467219820Sjeff } else { 468219820Sjeff perror("cmatose: failure in rdma_get_cm_event in disconnect events"); 469219820Sjeff ret = errno; 470219820Sjeff } 471219820Sjeff } 472219820Sjeff 473219820Sjeff return ret; 474219820Sjeff} 475219820Sjeff 476219820Sjeffstatic int migrate_channel(struct rdma_cm_id *listen_id) 477219820Sjeff{ 478219820Sjeff struct rdma_event_channel *channel; 479219820Sjeff int i, ret; 480219820Sjeff 481219820Sjeff printf("migrating to new event channel\n"); 482219820Sjeff 483219820Sjeff channel = rdma_create_event_channel(); 484219820Sjeff if (!channel) { 485219820Sjeff perror("cmatose: failed to create event channel"); 486219820Sjeff return -1; 487219820Sjeff } 488219820Sjeff 489219820Sjeff ret = 0; 490219820Sjeff if (listen_id) 491219820Sjeff ret = rdma_migrate_id(listen_id, channel); 492219820Sjeff 493219820Sjeff for (i = 0; i < connections && !ret; i++) 494219820Sjeff ret = rdma_migrate_id(test.nodes[i].cma_id, channel); 495219820Sjeff 496219820Sjeff if (!ret) { 497219820Sjeff rdma_destroy_event_channel(test.channel); 498219820Sjeff test.channel = channel; 499219820Sjeff } else 500219820Sjeff perror("cmatose: failure migrating to channel"); 501219820Sjeff 502219820Sjeff return ret; 503219820Sjeff} 504219820Sjeff 505219820Sjeffstatic int get_addr(char *dst, struct sockaddr *addr) 506219820Sjeff{ 507219820Sjeff struct addrinfo *res; 508219820Sjeff int ret; 509219820Sjeff 510219820Sjeff ret = getaddrinfo(dst, NULL, NULL, &res); 511219820Sjeff if (ret) { 512219820Sjeff printf("getaddrinfo failed - invalid hostname or IP address\n"); 513219820Sjeff return ret; 514219820Sjeff } 515219820Sjeff 516219820Sjeff if (res->ai_family == PF_INET) 517219820Sjeff memcpy(addr, res->ai_addr, sizeof(struct sockaddr_in)); 518219820Sjeff else if (res->ai_family == PF_INET6) 519219820Sjeff memcpy(addr, res->ai_addr, sizeof(struct sockaddr_in6)); 520219820Sjeff else 521219820Sjeff ret = -1; 522219820Sjeff 523219820Sjeff freeaddrinfo(res); 524219820Sjeff return ret; 525219820Sjeff} 526219820Sjeff 527219820Sjeffstatic int run_server(void) 528219820Sjeff{ 529219820Sjeff struct rdma_cm_id *listen_id; 530219820Sjeff int i, ret; 531219820Sjeff 532219820Sjeff printf("cmatose: starting server\n"); 533219820Sjeff ret = rdma_create_id(test.channel, &listen_id, &test, RDMA_PS_TCP); 534219820Sjeff if (ret) { 535219820Sjeff perror("cmatose: listen request failed"); 536219820Sjeff return ret; 537219820Sjeff } 538219820Sjeff 539219820Sjeff if (src_addr) { 540219820Sjeff ret = get_addr(src_addr, &test.addr.src_addr); 541219820Sjeff if (ret) 542219820Sjeff goto out; 543219820Sjeff if (test.addr.src_addr.sa_family == AF_INET) 544219820Sjeff ((struct sockaddr_in *) &test.addr.src_addr)->sin_port = port; 545219820Sjeff else 546219820Sjeff ((struct sockaddr_in6 *) &test.addr.src_addr)->sin6_port = port; 547219820Sjeff 548219820Sjeff } else { 549219820Sjeff test.addr.src_addr.sa_family = PF_INET; 550219820Sjeff ((struct sockaddr_in *) &test.addr.src_addr)->sin_port = port; 551219820Sjeff } 552219820Sjeff 553219820Sjeff ret = rdma_bind_addr(listen_id, &test.addr.src_addr); 554219820Sjeff 555219820Sjeff if (ret) { 556219820Sjeff perror("cmatose: bind address failed"); 557219820Sjeff goto out; 558219820Sjeff } 559219820Sjeff 560219820Sjeff ret = rdma_listen(listen_id, 0); 561219820Sjeff if (ret) { 562219820Sjeff perror("cmatose: failure trying to listen"); 563219820Sjeff goto out; 564219820Sjeff } 565219820Sjeff 566219820Sjeff ret = connect_events(); 567219820Sjeff if (ret) 568219820Sjeff goto out; 569219820Sjeff 570219820Sjeff if (message_count) { 571219820Sjeff printf("initiating data transfers\n"); 572219820Sjeff for (i = 0; i < connections; i++) { 573219820Sjeff ret = post_sends(&test.nodes[i]); 574219820Sjeff if (ret) 575219820Sjeff goto out; 576219820Sjeff } 577219820Sjeff 578219820Sjeff printf("completing sends\n"); 579219820Sjeff ret = poll_cqs(SEND_CQ_INDEX); 580219820Sjeff if (ret) 581219820Sjeff goto out; 582219820Sjeff 583219820Sjeff printf("receiving data transfers\n"); 584219820Sjeff ret = poll_cqs(RECV_CQ_INDEX); 585219820Sjeff if (ret) 586219820Sjeff goto out; 587219820Sjeff printf("data transfers complete\n"); 588219820Sjeff 589219820Sjeff } 590219820Sjeff 591219820Sjeff if (migrate) { 592219820Sjeff ret = migrate_channel(listen_id); 593219820Sjeff if (ret) 594219820Sjeff goto out; 595219820Sjeff } 596219820Sjeff 597219820Sjeff printf("cmatose: disconnecting\n"); 598219820Sjeff for (i = 0; i < connections; i++) { 599219820Sjeff if (!test.nodes[i].connected) 600219820Sjeff continue; 601219820Sjeff 602219820Sjeff test.nodes[i].connected = 0; 603219820Sjeff rdma_disconnect(test.nodes[i].cma_id); 604219820Sjeff } 605219820Sjeff 606219820Sjeff ret = disconnect_events(); 607219820Sjeff 608219820Sjeff printf("disconnected\n"); 609219820Sjeff 610219820Sjeffout: 611219820Sjeff rdma_destroy_id(listen_id); 612219820Sjeff return ret; 613219820Sjeff} 614219820Sjeff 615219820Sjeffstatic int run_client(void) 616219820Sjeff{ 617219820Sjeff int i, ret, ret2; 618219820Sjeff 619219820Sjeff printf("cmatose: starting client\n"); 620219820Sjeff if (src_addr) { 621219820Sjeff ret = get_addr(src_addr, &test.addr.src_addr); 622219820Sjeff if (ret) 623219820Sjeff return ret; 624219820Sjeff } 625219820Sjeff 626219820Sjeff ret = get_addr(dst_addr, &test.addr.dst_addr); 627219820Sjeff if (ret) 628219820Sjeff return ret; 629219820Sjeff 630219820Sjeff if (test.addr.dst_addr.sa_family == AF_INET) 631219820Sjeff ((struct sockaddr_in *) &test.addr.dst_addr)->sin_port = port; 632219820Sjeff else 633219820Sjeff ((struct sockaddr_in6 *) &test.addr.dst_addr)->sin6_port = port; 634219820Sjeff 635219820Sjeff printf("cmatose: connecting\n"); 636219820Sjeff for (i = 0; i < connections; i++) { 637219820Sjeff ret = rdma_resolve_addr(test.nodes[i].cma_id, 638219820Sjeff src_addr ? &test.addr.src_addr : NULL, 639219820Sjeff &test.addr.dst_addr, 2000); 640219820Sjeff if (ret) { 641219820Sjeff perror("cmatose: failure getting addr"); 642219820Sjeff connect_error(); 643219820Sjeff return ret; 644219820Sjeff } 645219820Sjeff } 646219820Sjeff 647219820Sjeff ret = connect_events(); 648219820Sjeff if (ret) 649219820Sjeff goto disc; 650219820Sjeff 651219820Sjeff if (message_count) { 652219820Sjeff printf("receiving data transfers\n"); 653219820Sjeff ret = poll_cqs(RECV_CQ_INDEX); 654219820Sjeff if (ret) 655219820Sjeff goto disc; 656219820Sjeff 657219820Sjeff printf("sending replies\n"); 658219820Sjeff for (i = 0; i < connections; i++) { 659219820Sjeff ret = post_sends(&test.nodes[i]); 660219820Sjeff if (ret) 661219820Sjeff goto disc; 662219820Sjeff } 663219820Sjeff 664219820Sjeff printf("data transfers complete\n"); 665219820Sjeff } 666219820Sjeff 667219820Sjeff ret = 0; 668219820Sjeff 669219820Sjeff if (migrate) { 670219820Sjeff ret = migrate_channel(NULL); 671219820Sjeff if (ret) 672219820Sjeff goto out; 673219820Sjeff } 674219820Sjeffdisc: 675219820Sjeff ret2 = disconnect_events(); 676219820Sjeff if (ret2) 677219820Sjeff ret = ret2; 678219820Sjeffout: 679219820Sjeff return ret; 680219820Sjeff} 681219820Sjeff 682219820Sjeffint main(int argc, char **argv) 683219820Sjeff{ 684219820Sjeff int op, ret; 685219820Sjeff 686219820Sjeff while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:m")) != -1) { 687219820Sjeff switch (op) { 688219820Sjeff case 's': 689219820Sjeff dst_addr = optarg; 690219820Sjeff break; 691219820Sjeff case 'b': 692219820Sjeff src_addr = optarg; 693219820Sjeff break; 694219820Sjeff case 'c': 695219820Sjeff connections = atoi(optarg); 696219820Sjeff break; 697219820Sjeff case 'C': 698219820Sjeff message_count = atoi(optarg); 699219820Sjeff break; 700219820Sjeff case 'S': 701219820Sjeff message_size = atoi(optarg); 702219820Sjeff break; 703219820Sjeff case 't': 704219820Sjeff set_tos = 1; 705219820Sjeff tos = (uint8_t) atoi(optarg); 706219820Sjeff break; 707219820Sjeff case 'p': 708219820Sjeff port = atoi(optarg); 709219820Sjeff break; 710219820Sjeff case 'm': 711219820Sjeff migrate = 1; 712219820Sjeff break; 713219820Sjeff default: 714219820Sjeff printf("usage: %s\n", argv[0]); 715219820Sjeff printf("\t[-s server_address]\n"); 716219820Sjeff printf("\t[-b bind_address]\n"); 717219820Sjeff printf("\t[-c connections]\n"); 718219820Sjeff printf("\t[-C message_count]\n"); 719219820Sjeff printf("\t[-S message_size]\n"); 720219820Sjeff printf("\t[-t type_of_service]\n"); 721219820Sjeff printf("\t[-p port_number]\n"); 722219820Sjeff printf("\t[-m(igrate)]\n"); 723219820Sjeff exit(1); 724219820Sjeff } 725219820Sjeff } 726219820Sjeff 727219820Sjeff test.connects_left = connections; 728219820Sjeff test.disconnects_left = connections; 729219820Sjeff 730219820Sjeff test.channel = rdma_create_event_channel(); 731219820Sjeff if (!test.channel) { 732219820Sjeff printf("failed to create event channel\n"); 733219820Sjeff exit(1); 734219820Sjeff } 735219820Sjeff 736219820Sjeff if (alloc_nodes()) 737219820Sjeff exit(1); 738219820Sjeff 739219820Sjeff if (dst_addr) 740219820Sjeff ret = run_client(); 741219820Sjeff else 742219820Sjeff ret = run_server(); 743219820Sjeff 744219820Sjeff printf("test complete\n"); 745219820Sjeff destroy_nodes(); 746219820Sjeff rdma_destroy_event_channel(test.channel); 747219820Sjeff 748219820Sjeff printf("return status %d\n", ret); 749219820Sjeff return ret; 750219820Sjeff} 751