krping.c revision 331769
1/* 2 * Copyright (c) 2005 Ammasso, Inc. All rights reserved. 3 * Copyright (c) 2006-2009 Open Grid Computing, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: stable/11/sys/contrib/rdma/krping/krping.c 331769 2018-03-30 18:06:29Z hselasky $"); 36 37#include <linux/module.h> 38#include <linux/moduleparam.h> 39#include <linux/slab.h> 40#include <linux/err.h> 41#include <linux/string.h> 42#include <linux/list.h> 43#include <linux/in.h> 44#include <linux/device.h> 45#include <linux/pci.h> 46#include <linux/sched.h> 47#include <linux/wait.h> 48 49#include <asm/atomic.h> 50 51#include <rdma/ib_verbs.h> 52#include <rdma/rdma_cm.h> 53 54#include "krping.h" 55#include "getopt.h" 56 57#define PFX "krping: " 58 59extern int krping_debug; 60#define DEBUG_LOG(...) do { if (krping_debug) log(LOG_INFO, __VA_ARGS__); } while (0) 61#define BIND_INFO 1 62 63MODULE_AUTHOR("Steve Wise"); 64MODULE_DESCRIPTION("RDMA ping server"); 65MODULE_LICENSE("Dual BSD/GPL"); 66MODULE_VERSION(krping, 1); 67MODULE_DEPEND(krping, linuxkpi, 1, 1, 1); 68 69static __inline uint64_t 70get_cycles(void) 71{ 72 uint32_t low, high; 73 __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 74 return (low | ((u_int64_t)high << 32)); 75} 76 77typedef uint64_t cycles_t; 78 79enum mem_type { 80 DMA = 1, 81 REG = 2, 82}; 83 84static const struct krping_option krping_opts[] = { 85 {"count", OPT_INT, 'C'}, 86 {"size", OPT_INT, 'S'}, 87 {"addr", OPT_STRING, 'a'}, 88 {"addr6", OPT_STRING, 'A'}, 89 {"port", OPT_INT, 'p'}, 90 {"verbose", OPT_NOPARAM, 'v'}, 91 {"validate", OPT_NOPARAM, 'V'}, 92 {"server", OPT_NOPARAM, 's'}, 93 {"client", OPT_NOPARAM, 'c'}, 94 {"server_inv", OPT_NOPARAM, 'I'}, 95 {"wlat", OPT_NOPARAM, 'l'}, 96 {"rlat", OPT_NOPARAM, 'L'}, 97 {"bw", OPT_NOPARAM, 'B'}, 98 {"duplex", OPT_NOPARAM, 'd'}, 99 {"txdepth", OPT_INT, 'T'}, 100 {"poll", OPT_NOPARAM, 'P'}, 101 {"local_dma_lkey", OPT_NOPARAM, 'Z'}, 102 {"read_inv", OPT_NOPARAM, 'R'}, 103 {"fr", OPT_NOPARAM, 'f'}, 104 {NULL, 0, 0} 105}; 106 107#define htonll(x) cpu_to_be64((x)) 108#define ntohll(x) cpu_to_be64((x)) 109 110static DEFINE_MUTEX(krping_mutex); 111 112/* 113 * List of running krping threads. 114 */ 115static LIST_HEAD(krping_cbs); 116 117/* 118 * Invoke like this, one on each side, using the server's address on 119 * the RDMA device (iw%d): 120 * 121 * /bin/echo server,port=9999,addr=192.168.69.142,validate > /proc/krping 122 * /bin/echo client,port=9999,addr=192.168.69.142,validate > /proc/krping 123 * /bin/echo client,port=9999,addr6=2001:db8:0:f101::1,validate > /proc/krping 124 * 125 * krping "ping/pong" loop: 126 * client sends source rkey/addr/len 127 * server receives source rkey/add/len 128 * server rdma reads "ping" data from source 129 * server sends "go ahead" on rdma read completion 130 * client sends sink rkey/addr/len 131 * server receives sink rkey/addr/len 132 * server rdma writes "pong" data to sink 133 * server sends "go ahead" on rdma write completion 134 * <repeat loop> 135 */ 136 137/* 138 * These states are used to signal events between the completion handler 139 * and the main client or server thread. 140 * 141 * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, 142 * and RDMA_WRITE_COMPLETE for each ping. 143 */ 144enum test_state { 145 IDLE = 1, 146 CONNECT_REQUEST, 147 ADDR_RESOLVED, 148 ROUTE_RESOLVED, 149 CONNECTED, 150 RDMA_READ_ADV, 151 RDMA_READ_COMPLETE, 152 RDMA_WRITE_ADV, 153 RDMA_WRITE_COMPLETE, 154 ERROR 155}; 156 157struct krping_rdma_info { 158 uint64_t buf; 159 uint32_t rkey; 160 uint32_t size; 161}; 162 163/* 164 * Default max buffer size for IO... 165 */ 166#define RPING_BUFSIZE 128*1024 167#define RPING_SQ_DEPTH 64 168 169/* 170 * Control block struct. 171 */ 172struct krping_cb { 173 int server; /* 0 iff client */ 174 struct ib_cq *cq; 175 struct ib_pd *pd; 176 struct ib_qp *qp; 177 178 struct ib_mr *dma_mr; 179 180 struct ib_fast_reg_page_list *page_list; 181 int page_list_len; 182 struct ib_reg_wr reg_mr_wr; 183 struct ib_send_wr invalidate_wr; 184 struct ib_mr *reg_mr; 185 int server_invalidate; 186 int read_inv; 187 u8 key; 188 189 struct ib_recv_wr rq_wr; /* recv work request record */ 190 struct ib_sge recv_sgl; /* recv single SGE */ 191 struct krping_rdma_info recv_buf __aligned(16); /* malloc'd buffer */ 192 u64 recv_dma_addr; 193 DECLARE_PCI_UNMAP_ADDR(recv_mapping) 194 195 struct ib_send_wr sq_wr; /* send work requrest record */ 196 struct ib_sge send_sgl; 197 struct krping_rdma_info send_buf __aligned(16); /* single send buf */ 198 u64 send_dma_addr; 199 DECLARE_PCI_UNMAP_ADDR(send_mapping) 200 201 struct ib_rdma_wr rdma_sq_wr; /* rdma work request record */ 202 struct ib_sge rdma_sgl; /* rdma single SGE */ 203 char *rdma_buf; /* used as rdma sink */ 204 u64 rdma_dma_addr; 205 DECLARE_PCI_UNMAP_ADDR(rdma_mapping) 206 struct ib_mr *rdma_mr; 207 208 uint32_t remote_rkey; /* remote guys RKEY */ 209 uint64_t remote_addr; /* remote guys TO */ 210 uint32_t remote_len; /* remote guys LEN */ 211 212 char *start_buf; /* rdma read src */ 213 u64 start_dma_addr; 214 DECLARE_PCI_UNMAP_ADDR(start_mapping) 215 struct ib_mr *start_mr; 216 217 enum test_state state; /* used for cond/signalling */ 218 wait_queue_head_t sem; 219 struct krping_stats stats; 220 221 uint16_t port; /* dst port in NBO */ 222 u8 addr[16] __aligned(8); /* dst addr in NBO */ 223 char *addr_str; /* dst addr string */ 224 uint8_t addr_type; /* ADDR_FAMILY - IPv4/V6 */ 225 int verbose; /* verbose logging */ 226 int count; /* ping count */ 227 int size; /* ping data size */ 228 int validate; /* validate ping data */ 229 int wlat; /* run wlat test */ 230 int rlat; /* run rlat test */ 231 int bw; /* run bw test */ 232 int duplex; /* run bw full duplex test */ 233 int poll; /* poll or block for rlat test */ 234 int txdepth; /* SQ depth */ 235 int local_dma_lkey; /* use 0 for lkey */ 236 int frtest; /* reg test */ 237 238 /* CM stuff */ 239 struct rdma_cm_id *cm_id; /* connection on client side,*/ 240 /* listener on server side. */ 241 struct rdma_cm_id *child_cm_id; /* connection on server side */ 242 struct list_head list; 243}; 244 245static int krping_cma_event_handler(struct rdma_cm_id *cma_id, 246 struct rdma_cm_event *event) 247{ 248 int ret; 249 struct krping_cb *cb = cma_id->context; 250 251 DEBUG_LOG("cma_event type %d cma_id %p (%s)\n", event->event, cma_id, 252 (cma_id == cb->cm_id) ? "parent" : "child"); 253 254 switch (event->event) { 255 case RDMA_CM_EVENT_ADDR_RESOLVED: 256 cb->state = ADDR_RESOLVED; 257 ret = rdma_resolve_route(cma_id, 2000); 258 if (ret) { 259 printk(KERN_ERR PFX "rdma_resolve_route error %d\n", 260 ret); 261 wake_up_interruptible(&cb->sem); 262 } 263 break; 264 265 case RDMA_CM_EVENT_ROUTE_RESOLVED: 266 cb->state = ROUTE_RESOLVED; 267 wake_up_interruptible(&cb->sem); 268 break; 269 270 case RDMA_CM_EVENT_CONNECT_REQUEST: 271 cb->state = CONNECT_REQUEST; 272 cb->child_cm_id = cma_id; 273 DEBUG_LOG("child cma %p\n", cb->child_cm_id); 274 wake_up_interruptible(&cb->sem); 275 break; 276 277 case RDMA_CM_EVENT_ESTABLISHED: 278 DEBUG_LOG("ESTABLISHED\n"); 279 if (!cb->server) { 280 cb->state = CONNECTED; 281 } 282 wake_up_interruptible(&cb->sem); 283 break; 284 285 case RDMA_CM_EVENT_ADDR_ERROR: 286 case RDMA_CM_EVENT_ROUTE_ERROR: 287 case RDMA_CM_EVENT_CONNECT_ERROR: 288 case RDMA_CM_EVENT_UNREACHABLE: 289 case RDMA_CM_EVENT_REJECTED: 290 printk(KERN_ERR PFX "cma event %d, error %d\n", event->event, 291 event->status); 292 cb->state = ERROR; 293 wake_up_interruptible(&cb->sem); 294 break; 295 296 case RDMA_CM_EVENT_DISCONNECTED: 297 printk(KERN_ERR PFX "DISCONNECT EVENT...\n"); 298 cb->state = ERROR; 299 wake_up_interruptible(&cb->sem); 300 break; 301 302 case RDMA_CM_EVENT_DEVICE_REMOVAL: 303 printk(KERN_ERR PFX "cma detected device removal!!!!\n"); 304 break; 305 306 default: 307 printk(KERN_ERR PFX "oof bad type!\n"); 308 wake_up_interruptible(&cb->sem); 309 break; 310 } 311 return 0; 312} 313 314static int server_recv(struct krping_cb *cb, struct ib_wc *wc) 315{ 316 if (wc->byte_len != sizeof(cb->recv_buf)) { 317 printk(KERN_ERR PFX "Received bogus data, size %d\n", 318 wc->byte_len); 319 return -1; 320 } 321 322 cb->remote_rkey = ntohl(cb->recv_buf.rkey); 323 cb->remote_addr = ntohll(cb->recv_buf.buf); 324 cb->remote_len = ntohl(cb->recv_buf.size); 325 DEBUG_LOG("Received rkey %x addr %llx len %d from peer\n", 326 cb->remote_rkey, (unsigned long long)cb->remote_addr, 327 cb->remote_len); 328 329 if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) 330 cb->state = RDMA_READ_ADV; 331 else 332 cb->state = RDMA_WRITE_ADV; 333 334 return 0; 335} 336 337static int client_recv(struct krping_cb *cb, struct ib_wc *wc) 338{ 339 if (wc->byte_len != sizeof(cb->recv_buf)) { 340 printk(KERN_ERR PFX "Received bogus data, size %d\n", 341 wc->byte_len); 342 return -1; 343 } 344 345 if (cb->state == RDMA_READ_ADV) 346 cb->state = RDMA_WRITE_ADV; 347 else 348 cb->state = RDMA_WRITE_COMPLETE; 349 350 return 0; 351} 352 353static void krping_cq_event_handler(struct ib_cq *cq, void *ctx) 354{ 355 struct krping_cb *cb = ctx; 356 struct ib_wc wc; 357 struct ib_recv_wr *bad_wr; 358 int ret; 359 360 BUG_ON(cb->cq != cq); 361 if (cb->state == ERROR) { 362 printk(KERN_ERR PFX "cq completion in ERROR state\n"); 363 return; 364 } 365 if (cb->frtest) { 366 printk(KERN_ERR PFX "cq completion event in frtest!\n"); 367 return; 368 } 369 if (!cb->wlat && !cb->rlat && !cb->bw) 370 ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 371 while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { 372 if (wc.status) { 373 if (wc.status == IB_WC_WR_FLUSH_ERR) { 374 DEBUG_LOG("cq flushed\n"); 375 continue; 376 } else { 377 printk(KERN_ERR PFX "cq completion failed with " 378 "wr_id %jx status %d opcode %d vender_err %x\n", 379 (uintmax_t)wc.wr_id, wc.status, wc.opcode, wc.vendor_err); 380 goto error; 381 } 382 } 383 384 switch (wc.opcode) { 385 case IB_WC_SEND: 386 DEBUG_LOG("send completion\n"); 387 cb->stats.send_bytes += cb->send_sgl.length; 388 cb->stats.send_msgs++; 389 break; 390 391 case IB_WC_RDMA_WRITE: 392 DEBUG_LOG("rdma write completion\n"); 393 cb->stats.write_bytes += cb->rdma_sq_wr.wr.sg_list->length; 394 cb->stats.write_msgs++; 395 cb->state = RDMA_WRITE_COMPLETE; 396 wake_up_interruptible(&cb->sem); 397 break; 398 399 case IB_WC_RDMA_READ: 400 DEBUG_LOG("rdma read completion\n"); 401 cb->stats.read_bytes += cb->rdma_sq_wr.wr.sg_list->length; 402 cb->stats.read_msgs++; 403 cb->state = RDMA_READ_COMPLETE; 404 wake_up_interruptible(&cb->sem); 405 break; 406 407 case IB_WC_RECV: 408 DEBUG_LOG("recv completion\n"); 409 cb->stats.recv_bytes += sizeof(cb->recv_buf); 410 cb->stats.recv_msgs++; 411 if (cb->wlat || cb->rlat || cb->bw) 412 ret = server_recv(cb, &wc); 413 else 414 ret = cb->server ? server_recv(cb, &wc) : 415 client_recv(cb, &wc); 416 if (ret) { 417 printk(KERN_ERR PFX "recv wc error: %d\n", ret); 418 goto error; 419 } 420 421 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 422 if (ret) { 423 printk(KERN_ERR PFX "post recv error: %d\n", 424 ret); 425 goto error; 426 } 427 wake_up_interruptible(&cb->sem); 428 break; 429 430 default: 431 printk(KERN_ERR PFX 432 "%s:%d Unexpected opcode %d, Shutting down\n", 433 __func__, __LINE__, wc.opcode); 434 goto error; 435 } 436 } 437 if (ret) { 438 printk(KERN_ERR PFX "poll error %d\n", ret); 439 goto error; 440 } 441 return; 442error: 443 cb->state = ERROR; 444 wake_up_interruptible(&cb->sem); 445} 446 447static int krping_accept(struct krping_cb *cb) 448{ 449 struct rdma_conn_param conn_param; 450 int ret; 451 452 DEBUG_LOG("accepting client connection request\n"); 453 454 memset(&conn_param, 0, sizeof conn_param); 455 conn_param.responder_resources = 1; 456 conn_param.initiator_depth = 1; 457 458 ret = rdma_accept(cb->child_cm_id, &conn_param); 459 if (ret) { 460 printk(KERN_ERR PFX "rdma_accept error: %d\n", ret); 461 return ret; 462 } 463 464 if (!cb->wlat && !cb->rlat && !cb->bw) { 465 wait_event_interruptible(cb->sem, cb->state >= CONNECTED); 466 if (cb->state == ERROR) { 467 printk(KERN_ERR PFX "wait for CONNECTED state %d\n", 468 cb->state); 469 return -1; 470 } 471 } 472 return 0; 473} 474 475static void krping_setup_wr(struct krping_cb *cb) 476{ 477 cb->recv_sgl.addr = cb->recv_dma_addr; 478 cb->recv_sgl.length = sizeof cb->recv_buf; 479 cb->recv_sgl.lkey = cb->pd->local_dma_lkey; 480 cb->rq_wr.sg_list = &cb->recv_sgl; 481 cb->rq_wr.num_sge = 1; 482 483 cb->send_sgl.addr = cb->send_dma_addr; 484 cb->send_sgl.length = sizeof cb->send_buf; 485 cb->send_sgl.lkey = cb->pd->local_dma_lkey; 486 487 cb->sq_wr.opcode = IB_WR_SEND; 488 cb->sq_wr.send_flags = IB_SEND_SIGNALED; 489 cb->sq_wr.sg_list = &cb->send_sgl; 490 cb->sq_wr.num_sge = 1; 491 492 if (cb->server || cb->wlat || cb->rlat || cb->bw) { 493 cb->rdma_sgl.addr = cb->rdma_dma_addr; 494 cb->rdma_sq_wr.wr.send_flags = IB_SEND_SIGNALED; 495 cb->rdma_sq_wr.wr.sg_list = &cb->rdma_sgl; 496 cb->rdma_sq_wr.wr.num_sge = 1; 497 } 498 499 /* 500 * A chain of 2 WRs, INVALDATE_MR + REG_MR. 501 * both unsignaled. The client uses them to reregister 502 * the rdma buffers with a new key each iteration. 503 */ 504 cb->reg_mr_wr.wr.opcode = IB_WR_REG_MR; 505 cb->reg_mr_wr.mr = cb->reg_mr; 506 507 cb->invalidate_wr.next = &cb->reg_mr_wr.wr; 508 cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; 509} 510 511static int krping_setup_buffers(struct krping_cb *cb) 512{ 513 int ret; 514 515 DEBUG_LOG(PFX "krping_setup_buffers called on cb %p\n", cb); 516 517 cb->recv_dma_addr = ib_dma_map_single(cb->pd->device, 518 &cb->recv_buf, 519 sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); 520 pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); 521 cb->send_dma_addr = ib_dma_map_single(cb->pd->device, 522 &cb->send_buf, sizeof(cb->send_buf), 523 DMA_BIDIRECTIONAL); 524 pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); 525 526 cb->rdma_buf = ib_dma_alloc_coherent(cb->pd->device, cb->size, 527 &cb->rdma_dma_addr, 528 GFP_KERNEL); 529 if (!cb->rdma_buf) { 530 DEBUG_LOG(PFX "rdma_buf allocation failed\n"); 531 ret = -ENOMEM; 532 goto bail; 533 } 534 pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); 535 cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + PAGE_SIZE) 536 >> PAGE_SHIFT; 537 cb->reg_mr = ib_alloc_mr(cb->pd, IB_MR_TYPE_MEM_REG, 538 cb->page_list_len); 539 if (IS_ERR(cb->reg_mr)) { 540 ret = PTR_ERR(cb->reg_mr); 541 DEBUG_LOG(PFX "recv_buf reg_mr failed %d\n", ret); 542 goto bail; 543 } 544 DEBUG_LOG(PFX "reg rkey 0x%x page_list_len %u\n", 545 cb->reg_mr->rkey, cb->page_list_len); 546 547 if (!cb->server || cb->wlat || cb->rlat || cb->bw) { 548 549 cb->start_buf = ib_dma_alloc_coherent(cb->pd->device, cb->size, 550 &cb->start_dma_addr, 551 GFP_KERNEL); 552 if (!cb->start_buf) { 553 DEBUG_LOG(PFX "start_buf malloc failed\n"); 554 ret = -ENOMEM; 555 goto bail; 556 } 557 pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); 558 } 559 560 krping_setup_wr(cb); 561 DEBUG_LOG(PFX "allocated & registered buffers...\n"); 562 return 0; 563bail: 564 if (cb->reg_mr && !IS_ERR(cb->reg_mr)) 565 ib_dereg_mr(cb->reg_mr); 566 if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) 567 ib_dereg_mr(cb->rdma_mr); 568 if (cb->dma_mr && !IS_ERR(cb->dma_mr)) 569 ib_dereg_mr(cb->dma_mr); 570 if (cb->rdma_buf) { 571 ib_dma_free_coherent(cb->pd->device, cb->size, cb->rdma_buf, 572 cb->rdma_dma_addr); 573 } 574 if (cb->start_buf) { 575 ib_dma_free_coherent(cb->pd->device, cb->size, cb->start_buf, 576 cb->start_dma_addr); 577 } 578 return ret; 579} 580 581static void krping_free_buffers(struct krping_cb *cb) 582{ 583 DEBUG_LOG("krping_free_buffers called on cb %p\n", cb); 584 585 if (cb->dma_mr) 586 ib_dereg_mr(cb->dma_mr); 587 if (cb->rdma_mr) 588 ib_dereg_mr(cb->rdma_mr); 589 if (cb->start_mr) 590 ib_dereg_mr(cb->start_mr); 591 if (cb->reg_mr) 592 ib_dereg_mr(cb->reg_mr); 593 594 dma_unmap_single(cb->pd->device->dma_device, 595 pci_unmap_addr(cb, recv_mapping), 596 sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); 597 dma_unmap_single(cb->pd->device->dma_device, 598 pci_unmap_addr(cb, send_mapping), 599 sizeof(cb->send_buf), DMA_BIDIRECTIONAL); 600 601 ib_dma_free_coherent(cb->pd->device, cb->size, cb->rdma_buf, 602 cb->rdma_dma_addr); 603 604 if (cb->start_buf) { 605 ib_dma_free_coherent(cb->pd->device, cb->size, cb->start_buf, 606 cb->start_dma_addr); 607 } 608} 609 610static int krping_create_qp(struct krping_cb *cb) 611{ 612 struct ib_qp_init_attr init_attr; 613 int ret; 614 615 memset(&init_attr, 0, sizeof(init_attr)); 616 init_attr.cap.max_send_wr = cb->txdepth; 617 init_attr.cap.max_recv_wr = 2; 618 619 /* For flush_qp() */ 620 init_attr.cap.max_send_wr++; 621 init_attr.cap.max_recv_wr++; 622 623 init_attr.cap.max_recv_sge = 1; 624 init_attr.cap.max_send_sge = 1; 625 init_attr.qp_type = IB_QPT_RC; 626 init_attr.send_cq = cb->cq; 627 init_attr.recv_cq = cb->cq; 628 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 629 630 if (cb->server) { 631 ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); 632 if (!ret) 633 cb->qp = cb->child_cm_id->qp; 634 } else { 635 ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); 636 if (!ret) 637 cb->qp = cb->cm_id->qp; 638 } 639 640 return ret; 641} 642 643static void krping_free_qp(struct krping_cb *cb) 644{ 645 ib_destroy_qp(cb->qp); 646 ib_destroy_cq(cb->cq); 647 ib_dealloc_pd(cb->pd); 648} 649 650static int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) 651{ 652 int ret; 653 struct ib_cq_init_attr attr = {0}; 654 655 cb->pd = ib_alloc_pd(cm_id->device, 0); 656 if (IS_ERR(cb->pd)) { 657 printk(KERN_ERR PFX "ib_alloc_pd failed\n"); 658 return PTR_ERR(cb->pd); 659 } 660 DEBUG_LOG("created pd %p\n", cb->pd); 661 662 strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); 663 664 attr.cqe = cb->txdepth * 2; 665 attr.comp_vector = 0; 666 cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, 667 cb, &attr); 668 if (IS_ERR(cb->cq)) { 669 printk(KERN_ERR PFX "ib_create_cq failed\n"); 670 ret = PTR_ERR(cb->cq); 671 goto err1; 672 } 673 DEBUG_LOG("created cq %p\n", cb->cq); 674 675 if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { 676 ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 677 if (ret) { 678 printk(KERN_ERR PFX "ib_create_cq failed\n"); 679 goto err2; 680 } 681 } 682 683 ret = krping_create_qp(cb); 684 if (ret) { 685 printk(KERN_ERR PFX "krping_create_qp failed: %d\n", ret); 686 goto err2; 687 } 688 DEBUG_LOG("created qp %p\n", cb->qp); 689 return 0; 690err2: 691 ib_destroy_cq(cb->cq); 692err1: 693 ib_dealloc_pd(cb->pd); 694 return ret; 695} 696 697/* 698 * return the (possibly rebound) rkey for the rdma buffer. 699 * REG mode: invalidate and rebind via reg wr. 700 * other modes: just return the mr rkey. 701 */ 702static u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) 703{ 704 u32 rkey; 705 struct ib_send_wr *bad_wr; 706 int ret; 707 struct scatterlist sg = {0}; 708 709 cb->invalidate_wr.ex.invalidate_rkey = cb->reg_mr->rkey; 710 711 /* 712 * Update the reg key. 713 */ 714 ib_update_fast_reg_key(cb->reg_mr, ++cb->key); 715 cb->reg_mr_wr.key = cb->reg_mr->rkey; 716 717 /* 718 * Update the reg WR with new buf info. 719 */ 720 if (buf == (u64)cb->start_dma_addr) 721 cb->reg_mr_wr.access = IB_ACCESS_REMOTE_READ; 722 else 723 cb->reg_mr_wr.access = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; 724 sg_dma_address(&sg) = buf; 725 sg_dma_len(&sg) = cb->size; 726 727 ret = ib_map_mr_sg(cb->reg_mr, &sg, 1, NULL, PAGE_SIZE); 728 BUG_ON(ret <= 0 || ret > cb->page_list_len); 729 730 DEBUG_LOG(PFX "post_inv = %d, reg_mr new rkey 0x%x pgsz %u len %u" 731 " iova_start %llx\n", 732 post_inv, 733 cb->reg_mr_wr.key, 734 cb->reg_mr->page_size, 735 cb->reg_mr->length, 736 (unsigned long long)cb->reg_mr->iova); 737 738 if (post_inv) 739 ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); 740 else 741 ret = ib_post_send(cb->qp, &cb->reg_mr_wr.wr, &bad_wr); 742 if (ret) { 743 printk(KERN_ERR PFX "post send error %d\n", ret); 744 cb->state = ERROR; 745 } 746 rkey = cb->reg_mr->rkey; 747 return rkey; 748} 749 750static void krping_format_send(struct krping_cb *cb, u64 buf) 751{ 752 struct krping_rdma_info *info = &cb->send_buf; 753 u32 rkey; 754 755 /* 756 * Client side will do reg or mw bind before 757 * advertising the rdma buffer. Server side 758 * sends have no data. 759 */ 760 if (!cb->server || cb->wlat || cb->rlat || cb->bw) { 761 rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); 762 info->buf = htonll(buf); 763 info->rkey = htonl(rkey); 764 info->size = htonl(cb->size); 765 DEBUG_LOG("RDMA addr %llx rkey %x len %d\n", 766 (unsigned long long)buf, rkey, cb->size); 767 } 768} 769 770static void krping_test_server(struct krping_cb *cb) 771{ 772 struct ib_send_wr *bad_wr, inv; 773 int ret; 774 775 while (1) { 776 /* Wait for client's Start STAG/TO/Len */ 777 wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); 778 if (cb->state != RDMA_READ_ADV) { 779 printk(KERN_ERR PFX "wait for RDMA_READ_ADV state %d\n", 780 cb->state); 781 break; 782 } 783 784 DEBUG_LOG("server received sink adv\n"); 785 786 cb->rdma_sq_wr.rkey = cb->remote_rkey; 787 cb->rdma_sq_wr.remote_addr = cb->remote_addr; 788 cb->rdma_sq_wr.wr.sg_list->length = cb->remote_len; 789 cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, !cb->read_inv); 790 cb->rdma_sq_wr.wr.next = NULL; 791 792 /* Issue RDMA Read. */ 793 if (cb->read_inv) 794 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV; 795 else { 796 797 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ; 798 /* 799 * Immediately follow the read with a 800 * fenced LOCAL_INV. 801 */ 802 cb->rdma_sq_wr.wr.next = &inv; 803 memset(&inv, 0, sizeof inv); 804 inv.opcode = IB_WR_LOCAL_INV; 805 inv.ex.invalidate_rkey = cb->reg_mr->rkey; 806 inv.send_flags = IB_SEND_FENCE; 807 } 808 809 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr); 810 if (ret) { 811 printk(KERN_ERR PFX "post send error %d\n", ret); 812 break; 813 } 814 cb->rdma_sq_wr.wr.next = NULL; 815 816 DEBUG_LOG("server posted rdma read req \n"); 817 818 /* Wait for read completion */ 819 wait_event_interruptible(cb->sem, 820 cb->state >= RDMA_READ_COMPLETE); 821 if (cb->state != RDMA_READ_COMPLETE) { 822 printk(KERN_ERR PFX 823 "wait for RDMA_READ_COMPLETE state %d\n", 824 cb->state); 825 break; 826 } 827 DEBUG_LOG("server received read complete\n"); 828 829 /* Display data in recv buf */ 830 if (cb->verbose) 831 printk(KERN_INFO PFX "server ping data: %s\n", 832 cb->rdma_buf); 833 834 /* Tell client to continue */ 835 if (cb->server && cb->server_invalidate) { 836 cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; 837 cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; 838 DEBUG_LOG("send-w-inv rkey 0x%x\n", cb->remote_rkey); 839 } 840 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 841 if (ret) { 842 printk(KERN_ERR PFX "post send error %d\n", ret); 843 break; 844 } 845 DEBUG_LOG("server posted go ahead\n"); 846 847 /* Wait for client's RDMA STAG/TO/Len */ 848 wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); 849 if (cb->state != RDMA_WRITE_ADV) { 850 printk(KERN_ERR PFX 851 "wait for RDMA_WRITE_ADV state %d\n", 852 cb->state); 853 break; 854 } 855 DEBUG_LOG("server received sink adv\n"); 856 857 /* RDMA Write echo data */ 858 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; 859 cb->rdma_sq_wr.rkey = cb->remote_rkey; 860 cb->rdma_sq_wr.remote_addr = cb->remote_addr; 861 cb->rdma_sq_wr.wr.sg_list->length = strlen(cb->rdma_buf) + 1; 862 if (cb->local_dma_lkey) 863 cb->rdma_sgl.lkey = cb->pd->local_dma_lkey; 864 else 865 cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); 866 867 DEBUG_LOG("rdma write from lkey %x laddr %llx len %d\n", 868 cb->rdma_sq_wr.wr.sg_list->lkey, 869 (unsigned long long)cb->rdma_sq_wr.wr.sg_list->addr, 870 cb->rdma_sq_wr.wr.sg_list->length); 871 872 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr); 873 if (ret) { 874 printk(KERN_ERR PFX "post send error %d\n", ret); 875 break; 876 } 877 878 /* Wait for completion */ 879 ret = wait_event_interruptible(cb->sem, cb->state >= 880 RDMA_WRITE_COMPLETE); 881 if (cb->state != RDMA_WRITE_COMPLETE) { 882 printk(KERN_ERR PFX 883 "wait for RDMA_WRITE_COMPLETE state %d\n", 884 cb->state); 885 break; 886 } 887 DEBUG_LOG("server rdma write complete \n"); 888 889 cb->state = CONNECTED; 890 891 /* Tell client to begin again */ 892 if (cb->server && cb->server_invalidate) { 893 cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; 894 cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; 895 DEBUG_LOG("send-w-inv rkey 0x%x\n", cb->remote_rkey); 896 } 897 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 898 if (ret) { 899 printk(KERN_ERR PFX "post send error %d\n", ret); 900 break; 901 } 902 DEBUG_LOG("server posted go ahead\n"); 903 } 904} 905 906static void rlat_test(struct krping_cb *cb) 907{ 908 int scnt; 909 int iters = cb->count; 910 struct timeval start_tv, stop_tv; 911 int ret; 912 struct ib_wc wc; 913 struct ib_send_wr *bad_wr; 914 int ne; 915 916 scnt = 0; 917 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ; 918 cb->rdma_sq_wr.rkey = cb->remote_rkey; 919 cb->rdma_sq_wr.remote_addr = cb->remote_addr; 920 cb->rdma_sq_wr.wr.sg_list->length = cb->size; 921 922 microtime(&start_tv); 923 if (!cb->poll) { 924 cb->state = RDMA_READ_ADV; 925 ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 926 } 927 while (scnt < iters) { 928 929 cb->state = RDMA_READ_ADV; 930 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr); 931 if (ret) { 932 printk(KERN_ERR PFX 933 "Couldn't post send: ret=%d scnt %d\n", 934 ret, scnt); 935 return; 936 } 937 938 do { 939 if (!cb->poll) { 940 wait_event_interruptible(cb->sem, 941 cb->state != RDMA_READ_ADV); 942 if (cb->state == RDMA_READ_COMPLETE) { 943 ne = 1; 944 ib_req_notify_cq(cb->cq, 945 IB_CQ_NEXT_COMP); 946 } else { 947 ne = -1; 948 } 949 } else 950 ne = ib_poll_cq(cb->cq, 1, &wc); 951 if (cb->state == ERROR) { 952 printk(KERN_ERR PFX 953 "state == ERROR...bailing scnt %d\n", 954 scnt); 955 return; 956 } 957 } while (ne == 0); 958 959 if (ne < 0) { 960 printk(KERN_ERR PFX "poll CQ failed %d\n", ne); 961 return; 962 } 963 if (cb->poll && wc.status != IB_WC_SUCCESS) { 964 printk(KERN_ERR PFX "Completion wth error at %s:\n", 965 cb->server ? "server" : "client"); 966 printk(KERN_ERR PFX "Failed status %d: wr_id %d\n", 967 wc.status, (int) wc.wr_id); 968 return; 969 } 970 ++scnt; 971 } 972 microtime(&stop_tv); 973 974 if (stop_tv.tv_usec < start_tv.tv_usec) { 975 stop_tv.tv_usec += 1000000; 976 stop_tv.tv_sec -= 1; 977 } 978 979 printk(KERN_ERR PFX "delta sec %lu delta usec %lu iter %d size %d\n", 980 (unsigned long)(stop_tv.tv_sec - start_tv.tv_sec), 981 (unsigned long)(stop_tv.tv_usec - start_tv.tv_usec), 982 scnt, cb->size); 983} 984 985static void wlat_test(struct krping_cb *cb) 986{ 987 int ccnt, scnt, rcnt; 988 int iters=cb->count; 989 volatile char *poll_buf = (char *) cb->start_buf; 990 char *buf = (char *)cb->rdma_buf; 991 struct timeval start_tv, stop_tv; 992 cycles_t *post_cycles_start, *post_cycles_stop; 993 cycles_t *poll_cycles_start, *poll_cycles_stop; 994 cycles_t *last_poll_cycles_start; 995 cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; 996 int i; 997 int cycle_iters = 1000; 998 999 ccnt = 0; 1000 scnt = 0; 1001 rcnt = 0; 1002 1003 post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1004 if (!post_cycles_start) { 1005 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1006 return; 1007 } 1008 post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1009 if (!post_cycles_stop) { 1010 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1011 return; 1012 } 1013 poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1014 if (!poll_cycles_start) { 1015 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1016 return; 1017 } 1018 poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1019 if (!poll_cycles_stop) { 1020 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1021 return; 1022 } 1023 last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), 1024 GFP_KERNEL); 1025 if (!last_poll_cycles_start) { 1026 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1027 return; 1028 } 1029 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; 1030 cb->rdma_sq_wr.rkey = cb->remote_rkey; 1031 cb->rdma_sq_wr.remote_addr = cb->remote_addr; 1032 cb->rdma_sq_wr.wr.sg_list->length = cb->size; 1033 1034 if (cycle_iters > iters) 1035 cycle_iters = iters; 1036 microtime(&start_tv); 1037 while (scnt < iters || ccnt < iters || rcnt < iters) { 1038 1039 /* Wait till buffer changes. */ 1040 if (rcnt < iters && !(scnt < 1 && !cb->server)) { 1041 ++rcnt; 1042 while (*poll_buf != (char)rcnt) { 1043 if (cb->state == ERROR) { 1044 printk(KERN_ERR PFX 1045 "state = ERROR, bailing\n"); 1046 return; 1047 } 1048 } 1049 } 1050 1051 if (scnt < iters) { 1052 struct ib_send_wr *bad_wr; 1053 1054 *buf = (char)scnt+1; 1055 if (scnt < cycle_iters) 1056 post_cycles_start[scnt] = get_cycles(); 1057 if (ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr)) { 1058 printk(KERN_ERR PFX 1059 "Couldn't post send: scnt=%d\n", 1060 scnt); 1061 return; 1062 } 1063 if (scnt < cycle_iters) 1064 post_cycles_stop[scnt] = get_cycles(); 1065 scnt++; 1066 } 1067 1068 if (ccnt < iters) { 1069 struct ib_wc wc; 1070 int ne; 1071 1072 if (ccnt < cycle_iters) 1073 poll_cycles_start[ccnt] = get_cycles(); 1074 do { 1075 if (ccnt < cycle_iters) 1076 last_poll_cycles_start[ccnt] = 1077 get_cycles(); 1078 ne = ib_poll_cq(cb->cq, 1, &wc); 1079 } while (ne == 0); 1080 if (ccnt < cycle_iters) 1081 poll_cycles_stop[ccnt] = get_cycles(); 1082 ++ccnt; 1083 1084 if (ne < 0) { 1085 printk(KERN_ERR PFX "poll CQ failed %d\n", ne); 1086 return; 1087 } 1088 if (wc.status != IB_WC_SUCCESS) { 1089 printk(KERN_ERR PFX 1090 "Completion wth error at %s:\n", 1091 cb->server ? "server" : "client"); 1092 printk(KERN_ERR PFX 1093 "Failed status %d: wr_id %d\n", 1094 wc.status, (int) wc.wr_id); 1095 printk(KERN_ERR PFX 1096 "scnt=%d, rcnt=%d, ccnt=%d\n", 1097 scnt, rcnt, ccnt); 1098 return; 1099 } 1100 } 1101 } 1102 microtime(&stop_tv); 1103 1104 if (stop_tv.tv_usec < start_tv.tv_usec) { 1105 stop_tv.tv_usec += 1000000; 1106 stop_tv.tv_sec -= 1; 1107 } 1108 1109 for (i=0; i < cycle_iters; i++) { 1110 sum_post += post_cycles_stop[i] - post_cycles_start[i]; 1111 sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; 1112 sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; 1113 } 1114 printk(KERN_ERR PFX 1115 "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" 1116 " sum_post %llu sum_poll %llu sum_last_poll %llu\n", 1117 (unsigned long)(stop_tv.tv_sec - start_tv.tv_sec), 1118 (unsigned long)(stop_tv.tv_usec - start_tv.tv_usec), 1119 scnt, cb->size, cycle_iters, 1120 (unsigned long long)sum_post, (unsigned long long)sum_poll, 1121 (unsigned long long)sum_last_poll); 1122 kfree(post_cycles_start); 1123 kfree(post_cycles_stop); 1124 kfree(poll_cycles_start); 1125 kfree(poll_cycles_stop); 1126 kfree(last_poll_cycles_start); 1127} 1128 1129static void bw_test(struct krping_cb *cb) 1130{ 1131 int ccnt, scnt, rcnt; 1132 int iters=cb->count; 1133 struct timeval start_tv, stop_tv; 1134 cycles_t *post_cycles_start, *post_cycles_stop; 1135 cycles_t *poll_cycles_start, *poll_cycles_stop; 1136 cycles_t *last_poll_cycles_start; 1137 cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; 1138 int i; 1139 int cycle_iters = 1000; 1140 1141 ccnt = 0; 1142 scnt = 0; 1143 rcnt = 0; 1144 1145 post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1146 if (!post_cycles_start) { 1147 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1148 return; 1149 } 1150 post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1151 if (!post_cycles_stop) { 1152 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1153 return; 1154 } 1155 poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1156 if (!poll_cycles_start) { 1157 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1158 return; 1159 } 1160 poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1161 if (!poll_cycles_stop) { 1162 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1163 return; 1164 } 1165 last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), 1166 GFP_KERNEL); 1167 if (!last_poll_cycles_start) { 1168 printk(KERN_ERR PFX "%s kmalloc failed\n", __FUNCTION__); 1169 return; 1170 } 1171 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; 1172 cb->rdma_sq_wr.rkey = cb->remote_rkey; 1173 cb->rdma_sq_wr.remote_addr = cb->remote_addr; 1174 cb->rdma_sq_wr.wr.sg_list->length = cb->size; 1175 1176 if (cycle_iters > iters) 1177 cycle_iters = iters; 1178 microtime(&start_tv); 1179 while (scnt < iters || ccnt < iters) { 1180 1181 while (scnt < iters && scnt - ccnt < cb->txdepth) { 1182 struct ib_send_wr *bad_wr; 1183 1184 if (scnt < cycle_iters) 1185 post_cycles_start[scnt] = get_cycles(); 1186 if (ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr)) { 1187 printk(KERN_ERR PFX 1188 "Couldn't post send: scnt=%d\n", 1189 scnt); 1190 return; 1191 } 1192 if (scnt < cycle_iters) 1193 post_cycles_stop[scnt] = get_cycles(); 1194 ++scnt; 1195 } 1196 1197 if (ccnt < iters) { 1198 int ne; 1199 struct ib_wc wc; 1200 1201 if (ccnt < cycle_iters) 1202 poll_cycles_start[ccnt] = get_cycles(); 1203 do { 1204 if (ccnt < cycle_iters) 1205 last_poll_cycles_start[ccnt] = 1206 get_cycles(); 1207 ne = ib_poll_cq(cb->cq, 1, &wc); 1208 } while (ne == 0); 1209 if (ccnt < cycle_iters) 1210 poll_cycles_stop[ccnt] = get_cycles(); 1211 ccnt += 1; 1212 1213 if (ne < 0) { 1214 printk(KERN_ERR PFX "poll CQ failed %d\n", ne); 1215 return; 1216 } 1217 if (wc.status != IB_WC_SUCCESS) { 1218 printk(KERN_ERR PFX 1219 "Completion wth error at %s:\n", 1220 cb->server ? "server" : "client"); 1221 printk(KERN_ERR PFX 1222 "Failed status %d: wr_id %d\n", 1223 wc.status, (int) wc.wr_id); 1224 return; 1225 } 1226 } 1227 } 1228 microtime(&stop_tv); 1229 1230 if (stop_tv.tv_usec < start_tv.tv_usec) { 1231 stop_tv.tv_usec += 1000000; 1232 stop_tv.tv_sec -= 1; 1233 } 1234 1235 for (i=0; i < cycle_iters; i++) { 1236 sum_post += post_cycles_stop[i] - post_cycles_start[i]; 1237 sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; 1238 sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; 1239 } 1240 printk(KERN_ERR PFX 1241 "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" 1242 " sum_post %llu sum_poll %llu sum_last_poll %llu\n", 1243 (unsigned long)(stop_tv.tv_sec - start_tv.tv_sec), 1244 (unsigned long)(stop_tv.tv_usec - start_tv.tv_usec), 1245 scnt, cb->size, cycle_iters, 1246 (unsigned long long)sum_post, (unsigned long long)sum_poll, 1247 (unsigned long long)sum_last_poll); 1248 kfree(post_cycles_start); 1249 kfree(post_cycles_stop); 1250 kfree(poll_cycles_start); 1251 kfree(poll_cycles_stop); 1252 kfree(last_poll_cycles_start); 1253} 1254 1255static void krping_rlat_test_server(struct krping_cb *cb) 1256{ 1257 struct ib_send_wr *bad_wr; 1258 struct ib_wc wc; 1259 int ret; 1260 1261 /* Spin waiting for client's Start STAG/TO/Len */ 1262 while (cb->state < RDMA_READ_ADV) { 1263 krping_cq_event_handler(cb->cq, cb); 1264 } 1265 1266 /* Send STAG/TO/Len to client */ 1267 krping_format_send(cb, cb->start_dma_addr); 1268 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1269 if (ret) { 1270 printk(KERN_ERR PFX "post send error %d\n", ret); 1271 return; 1272 } 1273 1274 /* Spin waiting for send completion */ 1275 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1276 if (ret < 0) { 1277 printk(KERN_ERR PFX "poll error %d\n", ret); 1278 return; 1279 } 1280 if (wc.status) { 1281 printk(KERN_ERR PFX "send completiong error %d\n", wc.status); 1282 return; 1283 } 1284 1285 wait_event_interruptible(cb->sem, cb->state == ERROR); 1286} 1287 1288static void krping_wlat_test_server(struct krping_cb *cb) 1289{ 1290 struct ib_send_wr *bad_wr; 1291 struct ib_wc wc; 1292 int ret; 1293 1294 /* Spin waiting for client's Start STAG/TO/Len */ 1295 while (cb->state < RDMA_READ_ADV) { 1296 krping_cq_event_handler(cb->cq, cb); 1297 } 1298 1299 /* Send STAG/TO/Len to client */ 1300 krping_format_send(cb, cb->start_dma_addr); 1301 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1302 if (ret) { 1303 printk(KERN_ERR PFX "post send error %d\n", ret); 1304 return; 1305 } 1306 1307 /* Spin waiting for send completion */ 1308 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1309 if (ret < 0) { 1310 printk(KERN_ERR PFX "poll error %d\n", ret); 1311 return; 1312 } 1313 if (wc.status) { 1314 printk(KERN_ERR PFX "send completiong error %d\n", wc.status); 1315 return; 1316 } 1317 1318 wlat_test(cb); 1319 wait_event_interruptible(cb->sem, cb->state == ERROR); 1320} 1321 1322static void krping_bw_test_server(struct krping_cb *cb) 1323{ 1324 struct ib_send_wr *bad_wr; 1325 struct ib_wc wc; 1326 int ret; 1327 1328 /* Spin waiting for client's Start STAG/TO/Len */ 1329 while (cb->state < RDMA_READ_ADV) { 1330 krping_cq_event_handler(cb->cq, cb); 1331 } 1332 1333 /* Send STAG/TO/Len to client */ 1334 krping_format_send(cb, cb->start_dma_addr); 1335 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1336 if (ret) { 1337 printk(KERN_ERR PFX "post send error %d\n", ret); 1338 return; 1339 } 1340 1341 /* Spin waiting for send completion */ 1342 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1343 if (ret < 0) { 1344 printk(KERN_ERR PFX "poll error %d\n", ret); 1345 return; 1346 } 1347 if (wc.status) { 1348 printk(KERN_ERR PFX "send completiong error %d\n", wc.status); 1349 return; 1350 } 1351 1352 if (cb->duplex) 1353 bw_test(cb); 1354 wait_event_interruptible(cb->sem, cb->state == ERROR); 1355} 1356 1357static int reg_supported(struct ib_device *dev) 1358{ 1359 u64 needed_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; 1360 1361 if ((dev->attrs.device_cap_flags & needed_flags) != needed_flags) { 1362 printk(KERN_ERR PFX 1363 "Fastreg not supported - device_cap_flags 0x%llx\n", 1364 (unsigned long long)dev->attrs.device_cap_flags); 1365 return 0; 1366 } 1367 DEBUG_LOG("Fastreg supported - device_cap_flags 0x%llx\n", 1368 (unsigned long long)dev->attrs.device_cap_flags); 1369 return 1; 1370} 1371 1372static void fill_sockaddr(struct sockaddr_storage *sin, struct krping_cb *cb) 1373{ 1374 memset(sin, 0, sizeof(*sin)); 1375 1376 if (cb->addr_type == AF_INET) { 1377 struct sockaddr_in *sin4 = (struct sockaddr_in *)sin; 1378 sin4->sin_len = sizeof(*sin4); 1379 sin4->sin_family = AF_INET; 1380 memcpy((void *)&sin4->sin_addr.s_addr, cb->addr, 4); 1381 sin4->sin_port = cb->port; 1382 } else if (cb->addr_type == AF_INET6) { 1383 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sin; 1384 sin6->sin6_len = sizeof(*sin6); 1385 sin6->sin6_family = AF_INET6; 1386 memcpy((void *)&sin6->sin6_addr, cb->addr, 16); 1387 sin6->sin6_port = cb->port; 1388 } 1389} 1390 1391static int krping_bind_server(struct krping_cb *cb) 1392{ 1393 struct sockaddr_storage sin; 1394 int ret; 1395 1396 1397 fill_sockaddr(&sin, cb); 1398 1399 ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *)&sin); 1400 if (ret) { 1401 printk(KERN_ERR PFX "rdma_bind_addr error %d\n", ret); 1402 return ret; 1403 } 1404 DEBUG_LOG("rdma_bind_addr successful\n"); 1405 1406 DEBUG_LOG("rdma_listen\n"); 1407 ret = rdma_listen(cb->cm_id, 3); 1408 if (ret) { 1409 printk(KERN_ERR PFX "rdma_listen failed: %d\n", ret); 1410 return ret; 1411 } 1412 1413 wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); 1414 if (cb->state != CONNECT_REQUEST) { 1415 printk(KERN_ERR PFX "wait for CONNECT_REQUEST state %d\n", 1416 cb->state); 1417 return -1; 1418 } 1419 1420 if (!reg_supported(cb->child_cm_id->device)) 1421 return -EINVAL; 1422 1423 return 0; 1424} 1425 1426static void krping_run_server(struct krping_cb *cb) 1427{ 1428 struct ib_recv_wr *bad_wr; 1429 int ret; 1430 1431 ret = krping_bind_server(cb); 1432 if (ret) 1433 return; 1434 1435 ret = krping_setup_qp(cb, cb->child_cm_id); 1436 if (ret) { 1437 printk(KERN_ERR PFX "setup_qp failed: %d\n", ret); 1438 goto err0; 1439 } 1440 1441 ret = krping_setup_buffers(cb); 1442 if (ret) { 1443 printk(KERN_ERR PFX "krping_setup_buffers failed: %d\n", ret); 1444 goto err1; 1445 } 1446 1447 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 1448 if (ret) { 1449 printk(KERN_ERR PFX "ib_post_recv failed: %d\n", ret); 1450 goto err2; 1451 } 1452 1453 ret = krping_accept(cb); 1454 if (ret) { 1455 printk(KERN_ERR PFX "connect error %d\n", ret); 1456 goto err2; 1457 } 1458 1459 if (cb->wlat) 1460 krping_wlat_test_server(cb); 1461 else if (cb->rlat) 1462 krping_rlat_test_server(cb); 1463 else if (cb->bw) 1464 krping_bw_test_server(cb); 1465 else 1466 krping_test_server(cb); 1467 rdma_disconnect(cb->child_cm_id); 1468err2: 1469 krping_free_buffers(cb); 1470err1: 1471 krping_free_qp(cb); 1472err0: 1473 rdma_destroy_id(cb->child_cm_id); 1474} 1475 1476static void krping_test_client(struct krping_cb *cb) 1477{ 1478 int ping, start, cc, i, ret; 1479 struct ib_send_wr *bad_wr; 1480 unsigned char c; 1481 1482 start = 65; 1483 for (ping = 0; !cb->count || ping < cb->count; ping++) { 1484 cb->state = RDMA_READ_ADV; 1485 1486 /* Put some ascii text in the buffer. */ 1487 cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); 1488 for (i = cc, c = start; i < cb->size; i++) { 1489 cb->start_buf[i] = c; 1490 c++; 1491 if (c > 122) 1492 c = 65; 1493 } 1494 start++; 1495 if (start > 122) 1496 start = 65; 1497 cb->start_buf[cb->size - 1] = 0; 1498 1499 krping_format_send(cb, cb->start_dma_addr); 1500 if (cb->state == ERROR) { 1501 printk(KERN_ERR PFX "krping_format_send failed\n"); 1502 break; 1503 } 1504 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1505 if (ret) { 1506 printk(KERN_ERR PFX "post send error %d\n", ret); 1507 break; 1508 } 1509 1510 /* Wait for server to ACK */ 1511 wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); 1512 if (cb->state != RDMA_WRITE_ADV) { 1513 printk(KERN_ERR PFX 1514 "wait for RDMA_WRITE_ADV state %d\n", 1515 cb->state); 1516 break; 1517 } 1518 1519 krping_format_send(cb, cb->rdma_dma_addr); 1520 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1521 if (ret) { 1522 printk(KERN_ERR PFX "post send error %d\n", ret); 1523 break; 1524 } 1525 1526 /* Wait for the server to say the RDMA Write is complete. */ 1527 wait_event_interruptible(cb->sem, 1528 cb->state >= RDMA_WRITE_COMPLETE); 1529 if (cb->state != RDMA_WRITE_COMPLETE) { 1530 printk(KERN_ERR PFX 1531 "wait for RDMA_WRITE_COMPLETE state %d\n", 1532 cb->state); 1533 break; 1534 } 1535 1536 if (cb->validate) 1537 if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { 1538 printk(KERN_ERR PFX "data mismatch!\n"); 1539 break; 1540 } 1541 1542 if (cb->verbose) 1543 printk(KERN_INFO PFX "ping data: %s\n", cb->rdma_buf); 1544#ifdef SLOW_KRPING 1545 wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); 1546#endif 1547 } 1548} 1549 1550static void krping_rlat_test_client(struct krping_cb *cb) 1551{ 1552 struct ib_send_wr *bad_wr; 1553 struct ib_wc wc; 1554 int ret; 1555 1556 cb->state = RDMA_READ_ADV; 1557 1558 /* Send STAG/TO/Len to client */ 1559 krping_format_send(cb, cb->start_dma_addr); 1560 if (cb->state == ERROR) { 1561 printk(KERN_ERR PFX "krping_format_send failed\n"); 1562 return; 1563 } 1564 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1565 if (ret) { 1566 printk(KERN_ERR PFX "post send error %d\n", ret); 1567 return; 1568 } 1569 1570 /* Spin waiting for send completion */ 1571 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1572 if (ret < 0) { 1573 printk(KERN_ERR PFX "poll error %d\n", ret); 1574 return; 1575 } 1576 if (wc.status) { 1577 printk(KERN_ERR PFX "send completion error %d\n", wc.status); 1578 return; 1579 } 1580 1581 /* Spin waiting for server's Start STAG/TO/Len */ 1582 while (cb->state < RDMA_WRITE_ADV) { 1583 krping_cq_event_handler(cb->cq, cb); 1584 } 1585 1586#if 0 1587{ 1588 int i; 1589 struct timeval start, stop; 1590 time_t sec; 1591 suseconds_t usec; 1592 unsigned long long elapsed; 1593 struct ib_wc wc; 1594 struct ib_send_wr *bad_wr; 1595 int ne; 1596 1597 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; 1598 cb->rdma_sq_wr.rkey = cb->remote_rkey; 1599 cb->rdma_sq_wr.remote_addr = cb->remote_addr; 1600 cb->rdma_sq_wr.wr.sg_list->length = 0; 1601 cb->rdma_sq_wr.wr.num_sge = 0; 1602 1603 microtime(&start); 1604 for (i=0; i < 100000; i++) { 1605 if (ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr)) { 1606 printk(KERN_ERR PFX "Couldn't post send\n"); 1607 return; 1608 } 1609 do { 1610 ne = ib_poll_cq(cb->cq, 1, &wc); 1611 } while (ne == 0); 1612 if (ne < 0) { 1613 printk(KERN_ERR PFX "poll CQ failed %d\n", ne); 1614 return; 1615 } 1616 if (wc.status != IB_WC_SUCCESS) { 1617 printk(KERN_ERR PFX "Completion wth error at %s:\n", 1618 cb->server ? "server" : "client"); 1619 printk(KERN_ERR PFX "Failed status %d: wr_id %d\n", 1620 wc.status, (int) wc.wr_id); 1621 return; 1622 } 1623 } 1624 microtime(&stop); 1625 1626 if (stop.tv_usec < start.tv_usec) { 1627 stop.tv_usec += 1000000; 1628 stop.tv_sec -= 1; 1629 } 1630 sec = stop.tv_sec - start.tv_sec; 1631 usec = stop.tv_usec - start.tv_usec; 1632 elapsed = sec * 1000000 + usec; 1633 printk(KERN_ERR PFX "0B-write-lat iters 100000 usec %llu\n", elapsed); 1634} 1635#endif 1636 1637 rlat_test(cb); 1638} 1639 1640static void krping_wlat_test_client(struct krping_cb *cb) 1641{ 1642 struct ib_send_wr *bad_wr; 1643 struct ib_wc wc; 1644 int ret; 1645 1646 cb->state = RDMA_READ_ADV; 1647 1648 /* Send STAG/TO/Len to client */ 1649 krping_format_send(cb, cb->start_dma_addr); 1650 if (cb->state == ERROR) { 1651 printk(KERN_ERR PFX "krping_format_send failed\n"); 1652 return; 1653 } 1654 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1655 if (ret) { 1656 printk(KERN_ERR PFX "post send error %d\n", ret); 1657 return; 1658 } 1659 1660 /* Spin waiting for send completion */ 1661 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1662 if (ret < 0) { 1663 printk(KERN_ERR PFX "poll error %d\n", ret); 1664 return; 1665 } 1666 if (wc.status) { 1667 printk(KERN_ERR PFX "send completion error %d\n", wc.status); 1668 return; 1669 } 1670 1671 /* Spin waiting for server's Start STAG/TO/Len */ 1672 while (cb->state < RDMA_WRITE_ADV) { 1673 krping_cq_event_handler(cb->cq, cb); 1674 } 1675 1676 wlat_test(cb); 1677} 1678 1679static void krping_bw_test_client(struct krping_cb *cb) 1680{ 1681 struct ib_send_wr *bad_wr; 1682 struct ib_wc wc; 1683 int ret; 1684 1685 cb->state = RDMA_READ_ADV; 1686 1687 /* Send STAG/TO/Len to client */ 1688 krping_format_send(cb, cb->start_dma_addr); 1689 if (cb->state == ERROR) { 1690 printk(KERN_ERR PFX "krping_format_send failed\n"); 1691 return; 1692 } 1693 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1694 if (ret) { 1695 printk(KERN_ERR PFX "post send error %d\n", ret); 1696 return; 1697 } 1698 1699 /* Spin waiting for send completion */ 1700 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1701 if (ret < 0) { 1702 printk(KERN_ERR PFX "poll error %d\n", ret); 1703 return; 1704 } 1705 if (wc.status) { 1706 printk(KERN_ERR PFX "send completion error %d\n", wc.status); 1707 return; 1708 } 1709 1710 /* Spin waiting for server's Start STAG/TO/Len */ 1711 while (cb->state < RDMA_WRITE_ADV) { 1712 krping_cq_event_handler(cb->cq, cb); 1713 } 1714 1715 bw_test(cb); 1716} 1717 1718/* 1719 * Manual qp flush test 1720 */ 1721static void flush_qp(struct krping_cb *cb) 1722{ 1723 struct ib_send_wr wr = { 0 }, *bad; 1724 struct ib_recv_wr recv_wr = { 0 }, *recv_bad; 1725 struct ib_wc wc; 1726 int ret; 1727 int flushed = 0; 1728 int ccnt = 0; 1729 1730 rdma_disconnect(cb->cm_id); 1731 DEBUG_LOG("disconnected!\n"); 1732 1733 wr.opcode = IB_WR_SEND; 1734 wr.wr_id = 0xdeadbeefcafebabe; 1735 ret = ib_post_send(cb->qp, &wr, &bad); 1736 if (ret) { 1737 printk(KERN_ERR PFX "%s post_send failed ret %d\n", __func__, ret); 1738 return; 1739 } 1740 1741 recv_wr.wr_id = 0xcafebabedeadbeef; 1742 ret = ib_post_recv(cb->qp, &recv_wr, &recv_bad); 1743 if (ret) { 1744 printk(KERN_ERR PFX "%s post_recv failed ret %d\n", __func__, ret); 1745 return; 1746 } 1747 1748 /* poll until the flush WRs complete */ 1749 do { 1750 ret = ib_poll_cq(cb->cq, 1, &wc); 1751 if (ret < 0) { 1752 printk(KERN_ERR PFX "ib_poll_cq failed %d\n", ret); 1753 return; 1754 } 1755 if (ret == 0) 1756 continue; 1757 ccnt++; 1758 if (wc.wr_id == 0xdeadbeefcafebabe || 1759 wc.wr_id == 0xcafebabedeadbeef) 1760 flushed++; 1761 } while (flushed != 2); 1762 DEBUG_LOG("qp_flushed! ccnt %u\n", ccnt); 1763} 1764 1765static void krping_fr_test(struct krping_cb *cb) 1766{ 1767 struct ib_send_wr inv, *bad; 1768 struct ib_reg_wr fr; 1769 struct ib_wc wc; 1770 u8 key = 0; 1771 struct ib_mr *mr; 1772 int ret; 1773 int size = cb->size; 1774 int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; 1775 unsigned long start; 1776 int count = 0; 1777 int scnt = 0; 1778 struct scatterlist sg = {0}; 1779 1780 mr = ib_alloc_mr(cb->pd, IB_MR_TYPE_MEM_REG, plen); 1781 if (IS_ERR(mr)) { 1782 printk(KERN_ERR PFX "ib_alloc_mr failed %ld\n", PTR_ERR(mr)); 1783 return; 1784 } 1785 1786 sg_dma_address(&sg) = (dma_addr_t)0xcafebabe0000ULL; 1787 sg_dma_len(&sg) = size; 1788 ret = ib_map_mr_sg(mr, &sg, 1, NULL, PAGE_SIZE); 1789 if (ret <= 0) { 1790 printk(KERN_ERR PFX "ib_map_mr_sge err %d\n", ret); 1791 goto err2; 1792 } 1793 1794 memset(&fr, 0, sizeof fr); 1795 fr.wr.opcode = IB_WR_REG_MR; 1796 fr.access = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; 1797 fr.mr = mr; 1798 fr.wr.next = &inv; 1799 1800 memset(&inv, 0, sizeof inv); 1801 inv.opcode = IB_WR_LOCAL_INV; 1802 inv.send_flags = IB_SEND_SIGNALED; 1803 1804 DEBUG_LOG("fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size, cb->txdepth); 1805 start = time_uptime; 1806 while (!cb->count || count <= cb->count) { 1807 if (SIGPENDING(curthread)) { 1808 printk(KERN_ERR PFX "signal!\n"); 1809 break; 1810 } 1811 if ((time_uptime - start) >= 9) { 1812 DEBUG_LOG("fr_test: pausing 1 second! count %u latest size %u plen %u\n", count, size, plen); 1813 wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); 1814 if (cb->state == ERROR) 1815 break; 1816 start = time_uptime; 1817 } 1818 while (scnt < (cb->txdepth>>1)) { 1819 ib_update_fast_reg_key(mr, ++key); 1820 fr.key = mr->rkey; 1821 inv.ex.invalidate_rkey = mr->rkey; 1822 1823 size = arc4random() % cb->size; 1824 if (size == 0) 1825 size = cb->size; 1826 sg_dma_len(&sg) = size; 1827 ret = ib_map_mr_sg(mr, &sg, 1, NULL, PAGE_SIZE); 1828 if (ret <= 0) { 1829 printk(KERN_ERR PFX "ib_map_mr_sge err %d\n", ret); 1830 goto err2; 1831 } 1832 ret = ib_post_send(cb->qp, &fr.wr, &bad); 1833 if (ret) { 1834 printk(KERN_ERR PFX "ib_post_send failed %d\n", ret); 1835 goto err2; 1836 } 1837 scnt++; 1838 } 1839 1840 ret = ib_poll_cq(cb->cq, 1, &wc); 1841 if (ret < 0) { 1842 printk(KERN_ERR PFX "ib_poll_cq failed %d\n", ret); 1843 goto err2; 1844 } 1845 if (ret == 1) { 1846 if (wc.status) { 1847 printk(KERN_ERR PFX "completion error %u\n", wc.status); 1848 goto err2; 1849 } 1850 count++; 1851 scnt--; 1852 } 1853 } 1854err2: 1855 flush_qp(cb); 1856 DEBUG_LOG("fr_test: done!\n"); 1857 ib_dereg_mr(mr); 1858} 1859 1860static int krping_connect_client(struct krping_cb *cb) 1861{ 1862 struct rdma_conn_param conn_param; 1863 int ret; 1864 1865 memset(&conn_param, 0, sizeof conn_param); 1866 conn_param.responder_resources = 1; 1867 conn_param.initiator_depth = 1; 1868 conn_param.retry_count = 10; 1869 1870 ret = rdma_connect(cb->cm_id, &conn_param); 1871 if (ret) { 1872 printk(KERN_ERR PFX "rdma_connect error %d\n", ret); 1873 return ret; 1874 } 1875 1876 wait_event_interruptible(cb->sem, cb->state >= CONNECTED); 1877 if (cb->state == ERROR) { 1878 printk(KERN_ERR PFX "wait for CONNECTED state %d\n", cb->state); 1879 return -1; 1880 } 1881 1882 DEBUG_LOG("rdma_connect successful\n"); 1883 return 0; 1884} 1885 1886static int krping_bind_client(struct krping_cb *cb) 1887{ 1888 struct sockaddr_storage sin; 1889 int ret; 1890 1891 fill_sockaddr(&sin, cb); 1892 1893 ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *)&sin, 2000); 1894 if (ret) { 1895 printk(KERN_ERR PFX "rdma_resolve_addr error %d\n", ret); 1896 return ret; 1897 } 1898 1899 wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); 1900 if (cb->state != ROUTE_RESOLVED) { 1901 printk(KERN_ERR PFX 1902 "addr/route resolution did not resolve: state %d\n", 1903 cb->state); 1904 return -EINTR; 1905 } 1906 1907 if (!reg_supported(cb->cm_id->device)) 1908 return -EINVAL; 1909 1910 DEBUG_LOG("rdma_resolve_addr - rdma_resolve_route successful\n"); 1911 return 0; 1912} 1913 1914static void krping_run_client(struct krping_cb *cb) 1915{ 1916 struct ib_recv_wr *bad_wr; 1917 int ret; 1918 1919 ret = krping_bind_client(cb); 1920 if (ret) 1921 return; 1922 1923 ret = krping_setup_qp(cb, cb->cm_id); 1924 if (ret) { 1925 printk(KERN_ERR PFX "setup_qp failed: %d\n", ret); 1926 return; 1927 } 1928 1929 ret = krping_setup_buffers(cb); 1930 if (ret) { 1931 printk(KERN_ERR PFX "krping_setup_buffers failed: %d\n", ret); 1932 goto err1; 1933 } 1934 1935 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 1936 if (ret) { 1937 printk(KERN_ERR PFX "ib_post_recv failed: %d\n", ret); 1938 goto err2; 1939 } 1940 1941 ret = krping_connect_client(cb); 1942 if (ret) { 1943 printk(KERN_ERR PFX "connect error %d\n", ret); 1944 goto err2; 1945 } 1946 1947 if (cb->wlat) 1948 krping_wlat_test_client(cb); 1949 else if (cb->rlat) 1950 krping_rlat_test_client(cb); 1951 else if (cb->bw) 1952 krping_bw_test_client(cb); 1953 else if (cb->frtest) 1954 krping_fr_test(cb); 1955 else 1956 krping_test_client(cb); 1957 rdma_disconnect(cb->cm_id); 1958err2: 1959 krping_free_buffers(cb); 1960err1: 1961 krping_free_qp(cb); 1962} 1963 1964static uint16_t 1965krping_get_ipv6_scope_id(char *name) 1966{ 1967 struct ifnet *ifp; 1968 uint16_t retval; 1969 1970 if (name == NULL) 1971 return (0); 1972 CURVNET_SET_QUIET(TD_TO_VNET(curthread)); 1973 ifp = ifunit_ref(name); 1974 CURVNET_RESTORE(); 1975 if (ifp == NULL) 1976 return (0); 1977 retval = ifp->if_index; 1978 if_rele(ifp); 1979 return (retval); 1980} 1981 1982int krping_doit(char *cmd) 1983{ 1984 struct krping_cb *cb; 1985 int op; 1986 int ret = 0; 1987 char *optarg; 1988 char *scope; 1989 unsigned long optint; 1990 1991 cb = kzalloc(sizeof(*cb), GFP_KERNEL); 1992 if (!cb) 1993 return -ENOMEM; 1994 1995 mutex_lock(&krping_mutex); 1996 list_add_tail(&cb->list, &krping_cbs); 1997 mutex_unlock(&krping_mutex); 1998 1999 cb->server = -1; 2000 cb->state = IDLE; 2001 cb->size = 64; 2002 cb->txdepth = RPING_SQ_DEPTH; 2003 init_waitqueue_head(&cb->sem); 2004 2005 while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, 2006 &optint)) != 0) { 2007 switch (op) { 2008 case 'a': 2009 cb->addr_str = optarg; 2010 cb->addr_type = AF_INET; 2011 DEBUG_LOG("ipaddr (%s)\n", optarg); 2012 if (inet_pton(AF_INET, optarg, cb->addr) != 1) { 2013 printk(KERN_ERR PFX "bad addr string %s\n", 2014 optarg); 2015 ret = EINVAL; 2016 } 2017 break; 2018 case 'A': 2019 cb->addr_str = optarg; 2020 cb->addr_type = AF_INET6; 2021 DEBUG_LOG("ipv6addr (%s)\n", optarg); 2022 scope = strstr(optarg, "%"); 2023 /* extract scope ID, if any */ 2024 if (scope != NULL) 2025 *scope++ = 0; 2026 /* extract IPv6 network address */ 2027 if (inet_pton(AF_INET6, optarg, cb->addr) != 1) { 2028 printk(KERN_ERR PFX "bad addr string %s\n", 2029 optarg); 2030 ret = EINVAL; 2031 } else if (IN6_IS_SCOPE_LINKLOCAL((struct in6_addr *)cb->addr) || 2032 IN6_IS_ADDR_MC_INTFACELOCAL((struct in6_addr *)cb->addr)) { 2033 uint16_t scope_id = krping_get_ipv6_scope_id(scope); 2034 DEBUG_LOG("ipv6 scope ID = %d\n", scope_id); 2035 cb->addr[2] = scope_id >> 8; 2036 cb->addr[3] = scope_id & 0xFF; 2037 } 2038 break; 2039 case 'p': 2040 cb->port = htons(optint); 2041 DEBUG_LOG("port %d\n", (int)optint); 2042 break; 2043 case 'P': 2044 cb->poll = 1; 2045 DEBUG_LOG("server\n"); 2046 break; 2047 case 's': 2048 cb->server = 1; 2049 DEBUG_LOG("server\n"); 2050 break; 2051 case 'c': 2052 cb->server = 0; 2053 DEBUG_LOG("client\n"); 2054 break; 2055 case 'S': 2056 cb->size = optint; 2057 if ((cb->size < 1) || 2058 (cb->size > RPING_BUFSIZE)) { 2059 printk(KERN_ERR PFX "Invalid size %d " 2060 "(valid range is 1 to %d)\n", 2061 cb->size, RPING_BUFSIZE); 2062 ret = EINVAL; 2063 } else 2064 DEBUG_LOG("size %d\n", (int)optint); 2065 break; 2066 case 'C': 2067 cb->count = optint; 2068 if (cb->count < 0) { 2069 printk(KERN_ERR PFX "Invalid count %d\n", 2070 cb->count); 2071 ret = EINVAL; 2072 } else 2073 DEBUG_LOG("count %d\n", (int) cb->count); 2074 break; 2075 case 'v': 2076 cb->verbose++; 2077 DEBUG_LOG("verbose\n"); 2078 break; 2079 case 'V': 2080 cb->validate++; 2081 DEBUG_LOG("validate data\n"); 2082 break; 2083 case 'l': 2084 cb->wlat++; 2085 break; 2086 case 'L': 2087 cb->rlat++; 2088 break; 2089 case 'B': 2090 cb->bw++; 2091 break; 2092 case 'd': 2093 cb->duplex++; 2094 break; 2095 case 'I': 2096 cb->server_invalidate = 1; 2097 break; 2098 case 'T': 2099 cb->txdepth = optint; 2100 DEBUG_LOG("txdepth %d\n", (int) cb->txdepth); 2101 break; 2102 case 'Z': 2103 cb->local_dma_lkey = 1; 2104 DEBUG_LOG("using local dma lkey\n"); 2105 break; 2106 case 'R': 2107 cb->read_inv = 1; 2108 DEBUG_LOG("using read-with-inv\n"); 2109 break; 2110 case 'f': 2111 cb->frtest = 1; 2112 DEBUG_LOG("fast-reg test!\n"); 2113 break; 2114 default: 2115 printk(KERN_ERR PFX "unknown opt %s\n", optarg); 2116 ret = -EINVAL; 2117 break; 2118 } 2119 } 2120 if (ret) 2121 goto out; 2122 2123 if (cb->server == -1) { 2124 printk(KERN_ERR PFX "must be either client or server\n"); 2125 ret = -EINVAL; 2126 goto out; 2127 } 2128 2129 if (cb->server && cb->frtest) { 2130 printk(KERN_ERR PFX "must be client to run frtest\n"); 2131 ret = -EINVAL; 2132 goto out; 2133 } 2134 2135 if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { 2136 printk(KERN_ERR PFX "Pick only one test: fr, bw, rlat, wlat\n"); 2137 ret = -EINVAL; 2138 goto out; 2139 } 2140 2141 if (cb->wlat || cb->rlat || cb->bw) { 2142 printk(KERN_ERR PFX "wlat, rlat, and bw tests only support mem_mode MR - which is no longer supported\n"); 2143 ret = -EINVAL; 2144 goto out; 2145 } 2146 2147 cb->cm_id = rdma_create_id(&init_net, krping_cma_event_handler, cb, RDMA_PS_TCP, IB_QPT_RC); 2148 if (IS_ERR(cb->cm_id)) { 2149 ret = PTR_ERR(cb->cm_id); 2150 printk(KERN_ERR PFX "rdma_create_id error %d\n", ret); 2151 goto out; 2152 } 2153 DEBUG_LOG("created cm_id %p\n", cb->cm_id); 2154 2155 if (cb->server) 2156 krping_run_server(cb); 2157 else 2158 krping_run_client(cb); 2159 2160 DEBUG_LOG("destroy cm_id %p\n", cb->cm_id); 2161 rdma_destroy_id(cb->cm_id); 2162out: 2163 mutex_lock(&krping_mutex); 2164 list_del(&cb->list); 2165 mutex_unlock(&krping_mutex); 2166 kfree(cb); 2167 return ret; 2168} 2169 2170void 2171krping_walk_cb_list(void (*f)(struct krping_stats *, void *), void *arg) 2172{ 2173 struct krping_cb *cb; 2174 2175 mutex_lock(&krping_mutex); 2176 list_for_each_entry(cb, &krping_cbs, list) 2177 (*f)(cb->pd ? &cb->stats : NULL, arg); 2178 mutex_unlock(&krping_mutex); 2179} 2180