1321936Shselasky/* 2321936Shselasky * Copyright (c) 2010-2014 Intel Corporation. All rights reserved. 3321936Shselasky * 4321936Shselasky * This software is available to you under a choice of one of two 5321936Shselasky * licenses. You may choose to be licensed under the terms of the GNU 6321936Shselasky * General Public License (GPL) Version 2, available from the file 7321936Shselasky * COPYING in the main directory of this source tree, or the 8321936Shselasky * OpenIB.org BSD license below: 9321936Shselasky * 10321936Shselasky * Redistribution and use in source and binary forms, with or 11321936Shselasky * without modification, are permitted provided that the following 12321936Shselasky * conditions are met: 13321936Shselasky * 14321936Shselasky * - Redistributions of source code must retain the above 15321936Shselasky * copyright notice, this list of conditions and the following 16321936Shselasky * disclaimer. 17321936Shselasky * 18321936Shselasky * - Redistributions in binary form must reproduce the above 19321936Shselasky * copyright notice, this list of conditions and the following 20321936Shselasky * disclaimer in the documentation and/or other materials 21321936Shselasky * provided with the distribution. 22321936Shselasky * 23321936Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24321936Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25321936Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26321936Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27321936Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28321936Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29321936Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30321936Shselasky * SOFTWARE. 31321936Shselasky */ 32321936Shselasky 33321936Shselasky#if !defined(RDMA_VERBS_H) 34321936Shselasky#define RDMA_VERBS_H 35321936Shselasky 36321936Shselasky#include <assert.h> 37321936Shselasky#include <infiniband/verbs.h> 38321936Shselasky#include <rdma/rdma_cma.h> 39321936Shselasky#include <errno.h> 40321936Shselasky 41321936Shselasky#ifdef __cplusplus 42321936Shselaskyextern "C" { 43321936Shselasky#endif 44321936Shselasky 45321936Shselaskystatic inline int rdma_seterrno(int ret) 46321936Shselasky{ 47321936Shselasky if (ret) { 48321936Shselasky errno = ret; 49321936Shselasky ret = -1; 50321936Shselasky } 51321936Shselasky return ret; 52321936Shselasky} 53321936Shselasky 54321936Shselasky/* 55321936Shselasky * Shared receive queues. 56321936Shselasky */ 57321936Shselaskyint rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd, 58321936Shselasky struct ibv_srq_init_attr *attr); 59321936Shselaskyint rdma_create_srq_ex(struct rdma_cm_id *id, struct ibv_srq_init_attr_ex *attr); 60321936Shselasky 61321936Shselaskyvoid rdma_destroy_srq(struct rdma_cm_id *id); 62321936Shselasky 63321936Shselasky 64321936Shselasky/* 65321936Shselasky * Memory registration helpers. 66321936Shselasky */ 67321936Shselaskystatic inline struct ibv_mr * 68321936Shselaskyrdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length) 69321936Shselasky{ 70321936Shselasky return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE); 71321936Shselasky} 72321936Shselasky 73321936Shselaskystatic inline struct ibv_mr * 74321936Shselaskyrdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length) 75321936Shselasky{ 76321936Shselasky return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE | 77321936Shselasky IBV_ACCESS_REMOTE_READ); 78321936Shselasky} 79321936Shselasky 80321936Shselaskystatic inline struct ibv_mr * 81321936Shselaskyrdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length) 82321936Shselasky{ 83321936Shselasky return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE | 84321936Shselasky IBV_ACCESS_REMOTE_WRITE); 85321936Shselasky} 86321936Shselasky 87321936Shselaskystatic inline int 88321936Shselaskyrdma_dereg_mr(struct ibv_mr *mr) 89321936Shselasky{ 90321936Shselasky return rdma_seterrno(ibv_dereg_mr(mr)); 91321936Shselasky} 92321936Shselasky 93321936Shselasky 94321936Shselasky/* 95321936Shselasky * Vectored send, receive, and RDMA operations. 96321936Shselasky * Support multiple scatter-gather entries. 97321936Shselasky */ 98321936Shselaskystatic inline int 99321936Shselaskyrdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 100321936Shselasky int nsge) 101321936Shselasky{ 102321936Shselasky struct ibv_recv_wr wr, *bad; 103321936Shselasky 104321936Shselasky wr.wr_id = (uintptr_t) context; 105321936Shselasky wr.next = NULL; 106321936Shselasky wr.sg_list = sgl; 107321936Shselasky wr.num_sge = nsge; 108321936Shselasky 109321936Shselasky if (id->srq) 110321936Shselasky return rdma_seterrno(ibv_post_srq_recv(id->srq, &wr, &bad)); 111321936Shselasky else 112321936Shselasky return rdma_seterrno(ibv_post_recv(id->qp, &wr, &bad)); 113321936Shselasky} 114321936Shselasky 115321936Shselaskystatic inline int 116321936Shselaskyrdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 117321936Shselasky int nsge, int flags) 118321936Shselasky{ 119321936Shselasky struct ibv_send_wr wr, *bad; 120321936Shselasky 121321936Shselasky wr.wr_id = (uintptr_t) context; 122321936Shselasky wr.next = NULL; 123321936Shselasky wr.sg_list = sgl; 124321936Shselasky wr.num_sge = nsge; 125321936Shselasky wr.opcode = IBV_WR_SEND; 126321936Shselasky wr.send_flags = flags; 127321936Shselasky 128321936Shselasky return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 129321936Shselasky} 130321936Shselasky 131321936Shselaskystatic inline int 132321936Shselaskyrdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 133321936Shselasky int nsge, int flags, uint64_t remote_addr, uint32_t rkey) 134321936Shselasky{ 135321936Shselasky struct ibv_send_wr wr, *bad; 136321936Shselasky 137321936Shselasky wr.wr_id = (uintptr_t) context; 138321936Shselasky wr.next = NULL; 139321936Shselasky wr.sg_list = sgl; 140321936Shselasky wr.num_sge = nsge; 141321936Shselasky wr.opcode = IBV_WR_RDMA_READ; 142321936Shselasky wr.send_flags = flags; 143321936Shselasky wr.wr.rdma.remote_addr = remote_addr; 144321936Shselasky wr.wr.rdma.rkey = rkey; 145321936Shselasky 146321936Shselasky return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 147321936Shselasky} 148321936Shselasky 149321936Shselaskystatic inline int 150321936Shselaskyrdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 151321936Shselasky int nsge, int flags, uint64_t remote_addr, uint32_t rkey) 152321936Shselasky{ 153321936Shselasky struct ibv_send_wr wr, *bad; 154321936Shselasky 155321936Shselasky wr.wr_id = (uintptr_t) context; 156321936Shselasky wr.next = NULL; 157321936Shselasky wr.sg_list = sgl; 158321936Shselasky wr.num_sge = nsge; 159321936Shselasky wr.opcode = IBV_WR_RDMA_WRITE; 160321936Shselasky wr.send_flags = flags; 161321936Shselasky wr.wr.rdma.remote_addr = remote_addr; 162321936Shselasky wr.wr.rdma.rkey = rkey; 163321936Shselasky 164321936Shselasky return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 165321936Shselasky} 166321936Shselasky 167321936Shselasky/* 168321936Shselasky * Simple send, receive, and RDMA calls. 169321936Shselasky */ 170321936Shselaskystatic inline int 171321936Shselaskyrdma_post_recv(struct rdma_cm_id *id, void *context, void *addr, 172321936Shselasky size_t length, struct ibv_mr *mr) 173321936Shselasky{ 174321936Shselasky struct ibv_sge sge; 175321936Shselasky 176321936Shselasky assert((addr >= mr->addr) && 177321936Shselasky (((uint8_t *) addr + length) <= ((uint8_t *) mr->addr + mr->length))); 178321936Shselasky sge.addr = (uint64_t) (uintptr_t) addr; 179321936Shselasky sge.length = (uint32_t) length; 180321936Shselasky sge.lkey = mr->lkey; 181321936Shselasky 182321936Shselasky return rdma_post_recvv(id, context, &sge, 1); 183321936Shselasky} 184321936Shselasky 185321936Shselaskystatic inline int 186321936Shselaskyrdma_post_send(struct rdma_cm_id *id, void *context, void *addr, 187321936Shselasky size_t length, struct ibv_mr *mr, int flags) 188321936Shselasky{ 189321936Shselasky struct ibv_sge sge; 190321936Shselasky 191321936Shselasky sge.addr = (uint64_t) (uintptr_t) addr; 192321936Shselasky sge.length = (uint32_t) length; 193321936Shselasky sge.lkey = mr ? mr->lkey : 0; 194321936Shselasky 195321936Shselasky return rdma_post_sendv(id, context, &sge, 1, flags); 196321936Shselasky} 197321936Shselasky 198321936Shselaskystatic inline int 199321936Shselaskyrdma_post_read(struct rdma_cm_id *id, void *context, void *addr, 200321936Shselasky size_t length, struct ibv_mr *mr, int flags, 201321936Shselasky uint64_t remote_addr, uint32_t rkey) 202321936Shselasky{ 203321936Shselasky struct ibv_sge sge; 204321936Shselasky 205321936Shselasky sge.addr = (uint64_t) (uintptr_t) addr; 206321936Shselasky sge.length = (uint32_t) length; 207321936Shselasky sge.lkey = mr->lkey; 208321936Shselasky 209321936Shselasky return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey); 210321936Shselasky} 211321936Shselasky 212321936Shselaskystatic inline int 213321936Shselaskyrdma_post_write(struct rdma_cm_id *id, void *context, void *addr, 214321936Shselasky size_t length, struct ibv_mr *mr, int flags, 215321936Shselasky uint64_t remote_addr, uint32_t rkey) 216321936Shselasky{ 217321936Shselasky struct ibv_sge sge; 218321936Shselasky 219321936Shselasky sge.addr = (uint64_t) (uintptr_t) addr; 220321936Shselasky sge.length = (uint32_t) length; 221321936Shselasky sge.lkey = mr ? mr->lkey : 0; 222321936Shselasky 223321936Shselasky return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey); 224321936Shselasky} 225321936Shselasky 226321936Shselaskystatic inline int 227321936Shselaskyrdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr, 228321936Shselasky size_t length, struct ibv_mr *mr, int flags, 229321936Shselasky struct ibv_ah *ah, uint32_t remote_qpn) 230321936Shselasky{ 231321936Shselasky struct ibv_send_wr wr, *bad; 232321936Shselasky struct ibv_sge sge; 233321936Shselasky 234321936Shselasky sge.addr = (uint64_t) (uintptr_t) addr; 235321936Shselasky sge.length = (uint32_t) length; 236321936Shselasky sge.lkey = mr ? mr->lkey : 0; 237321936Shselasky 238321936Shselasky wr.wr_id = (uintptr_t) context; 239321936Shselasky wr.next = NULL; 240321936Shselasky wr.sg_list = &sge; 241321936Shselasky wr.num_sge = 1; 242321936Shselasky wr.opcode = IBV_WR_SEND; 243321936Shselasky wr.send_flags = flags; 244321936Shselasky wr.wr.ud.ah = ah; 245321936Shselasky wr.wr.ud.remote_qpn = remote_qpn; 246321936Shselasky wr.wr.ud.remote_qkey = RDMA_UDP_QKEY; 247321936Shselasky 248321936Shselasky return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 249321936Shselasky} 250321936Shselasky 251321936Shselaskystatic inline int 252321936Shselaskyrdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc) 253321936Shselasky{ 254321936Shselasky struct ibv_cq *cq; 255321936Shselasky void *context; 256321936Shselasky int ret; 257321936Shselasky 258321936Shselasky do { 259321936Shselasky ret = ibv_poll_cq(id->send_cq, 1, wc); 260321936Shselasky if (ret) 261321936Shselasky break; 262321936Shselasky 263321936Shselasky ret = ibv_req_notify_cq(id->send_cq, 0); 264321936Shselasky if (ret) 265321936Shselasky return rdma_seterrno(ret); 266321936Shselasky 267321936Shselasky ret = ibv_poll_cq(id->send_cq, 1, wc); 268321936Shselasky if (ret) 269321936Shselasky break; 270321936Shselasky 271321936Shselasky ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context); 272321936Shselasky if (ret) 273321936Shselasky return ret; 274321936Shselasky 275321936Shselasky assert(cq == id->send_cq && context == id); 276321936Shselasky ibv_ack_cq_events(id->send_cq, 1); 277321936Shselasky } while (1); 278321936Shselasky 279321936Shselasky return (ret < 0) ? rdma_seterrno(ret) : ret; 280321936Shselasky} 281321936Shselasky 282321936Shselaskystatic inline int 283321936Shselaskyrdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc) 284321936Shselasky{ 285321936Shselasky struct ibv_cq *cq; 286321936Shselasky void *context; 287321936Shselasky int ret; 288321936Shselasky 289321936Shselasky do { 290321936Shselasky ret = ibv_poll_cq(id->recv_cq, 1, wc); 291321936Shselasky if (ret) 292321936Shselasky break; 293321936Shselasky 294321936Shselasky ret = ibv_req_notify_cq(id->recv_cq, 0); 295321936Shselasky if (ret) 296321936Shselasky return rdma_seterrno(ret); 297321936Shselasky 298321936Shselasky ret = ibv_poll_cq(id->recv_cq, 1, wc); 299321936Shselasky if (ret) 300321936Shselasky break; 301321936Shselasky 302321936Shselasky ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context); 303321936Shselasky if (ret) 304321936Shselasky return ret; 305321936Shselasky 306321936Shselasky assert(cq == id->recv_cq && context == id); 307321936Shselasky ibv_ack_cq_events(id->recv_cq, 1); 308321936Shselasky } while (1); 309321936Shselasky 310321936Shselasky return (ret < 0) ? rdma_seterrno(ret) : ret; 311321936Shselasky} 312321936Shselasky 313321936Shselasky#ifdef __cplusplus 314321936Shselasky} 315321936Shselasky#endif 316321936Shselasky 317321936Shselasky#endif /* RDMA_CMA_H */ 318