1/* 2 * Copyright (c) 2010-2014 Intel Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#if !defined(RDMA_VERBS_H) 34#define RDMA_VERBS_H 35 36#include <assert.h> 37#include <infiniband/verbs.h> 38#include <rdma/rdma_cma.h> 39#include <errno.h> 40 41#ifdef __cplusplus 42extern "C" { 43#endif 44 45static inline int rdma_seterrno(int ret) 46{ 47 if (ret) { 48 errno = ret; 49 ret = -1; 50 } 51 return ret; 52} 53 54/* 55 * Shared receive queues. 56 */ 57int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd, 58 struct ibv_srq_init_attr *attr); 59int rdma_create_srq_ex(struct rdma_cm_id *id, struct ibv_srq_init_attr_ex *attr); 60 61void rdma_destroy_srq(struct rdma_cm_id *id); 62 63 64/* 65 * Memory registration helpers. 66 */ 67static inline struct ibv_mr * 68rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length) 69{ 70 return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE); 71} 72 73static inline struct ibv_mr * 74rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length) 75{ 76 return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE | 77 IBV_ACCESS_REMOTE_READ); 78} 79 80static inline struct ibv_mr * 81rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length) 82{ 83 return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE | 84 IBV_ACCESS_REMOTE_WRITE); 85} 86 87static inline int 88rdma_dereg_mr(struct ibv_mr *mr) 89{ 90 return rdma_seterrno(ibv_dereg_mr(mr)); 91} 92 93 94/* 95 * Vectored send, receive, and RDMA operations. 96 * Support multiple scatter-gather entries. 97 */ 98static inline int 99rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 100 int nsge) 101{ 102 struct ibv_recv_wr wr, *bad; 103 104 wr.wr_id = (uintptr_t) context; 105 wr.next = NULL; 106 wr.sg_list = sgl; 107 wr.num_sge = nsge; 108 109 if (id->srq) 110 return rdma_seterrno(ibv_post_srq_recv(id->srq, &wr, &bad)); 111 else 112 return rdma_seterrno(ibv_post_recv(id->qp, &wr, &bad)); 113} 114 115static inline int 116rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 117 int nsge, int flags) 118{ 119 struct ibv_send_wr wr, *bad; 120 121 wr.wr_id = (uintptr_t) context; 122 wr.next = NULL; 123 wr.sg_list = sgl; 124 wr.num_sge = nsge; 125 wr.opcode = IBV_WR_SEND; 126 wr.send_flags = flags; 127 128 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 129} 130 131static inline int 132rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 133 int nsge, int flags, uint64_t remote_addr, uint32_t rkey) 134{ 135 struct ibv_send_wr wr, *bad; 136 137 wr.wr_id = (uintptr_t) context; 138 wr.next = NULL; 139 wr.sg_list = sgl; 140 wr.num_sge = nsge; 141 wr.opcode = IBV_WR_RDMA_READ; 142 wr.send_flags = flags; 143 wr.wr.rdma.remote_addr = remote_addr; 144 wr.wr.rdma.rkey = rkey; 145 146 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 147} 148 149static inline int 150rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl, 151 int nsge, int flags, uint64_t remote_addr, uint32_t rkey) 152{ 153 struct ibv_send_wr wr, *bad; 154 155 wr.wr_id = (uintptr_t) context; 156 wr.next = NULL; 157 wr.sg_list = sgl; 158 wr.num_sge = nsge; 159 wr.opcode = IBV_WR_RDMA_WRITE; 160 wr.send_flags = flags; 161 wr.wr.rdma.remote_addr = remote_addr; 162 wr.wr.rdma.rkey = rkey; 163 164 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 165} 166 167/* 168 * Simple send, receive, and RDMA calls. 169 */ 170static inline int 171rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr, 172 size_t length, struct ibv_mr *mr) 173{ 174 struct ibv_sge sge; 175 176 assert((addr >= mr->addr) && 177 (((uint8_t *) addr + length) <= ((uint8_t *) mr->addr + mr->length))); 178 sge.addr = (uint64_t) (uintptr_t) addr; 179 sge.length = (uint32_t) length; 180 sge.lkey = mr->lkey; 181 182 return rdma_post_recvv(id, context, &sge, 1); 183} 184 185static inline int 186rdma_post_send(struct rdma_cm_id *id, void *context, void *addr, 187 size_t length, struct ibv_mr *mr, int flags) 188{ 189 struct ibv_sge sge; 190 191 sge.addr = (uint64_t) (uintptr_t) addr; 192 sge.length = (uint32_t) length; 193 sge.lkey = mr ? mr->lkey : 0; 194 195 return rdma_post_sendv(id, context, &sge, 1, flags); 196} 197 198static inline int 199rdma_post_read(struct rdma_cm_id *id, void *context, void *addr, 200 size_t length, struct ibv_mr *mr, int flags, 201 uint64_t remote_addr, uint32_t rkey) 202{ 203 struct ibv_sge sge; 204 205 sge.addr = (uint64_t) (uintptr_t) addr; 206 sge.length = (uint32_t) length; 207 sge.lkey = mr->lkey; 208 209 return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey); 210} 211 212static inline int 213rdma_post_write(struct rdma_cm_id *id, void *context, void *addr, 214 size_t length, struct ibv_mr *mr, int flags, 215 uint64_t remote_addr, uint32_t rkey) 216{ 217 struct ibv_sge sge; 218 219 sge.addr = (uint64_t) (uintptr_t) addr; 220 sge.length = (uint32_t) length; 221 sge.lkey = mr ? mr->lkey : 0; 222 223 return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey); 224} 225 226static inline int 227rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr, 228 size_t length, struct ibv_mr *mr, int flags, 229 struct ibv_ah *ah, uint32_t remote_qpn) 230{ 231 struct ibv_send_wr wr, *bad; 232 struct ibv_sge sge; 233 234 sge.addr = (uint64_t) (uintptr_t) addr; 235 sge.length = (uint32_t) length; 236 sge.lkey = mr ? mr->lkey : 0; 237 238 wr.wr_id = (uintptr_t) context; 239 wr.next = NULL; 240 wr.sg_list = &sge; 241 wr.num_sge = 1; 242 wr.opcode = IBV_WR_SEND; 243 wr.send_flags = flags; 244 wr.wr.ud.ah = ah; 245 wr.wr.ud.remote_qpn = remote_qpn; 246 wr.wr.ud.remote_qkey = RDMA_UDP_QKEY; 247 248 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad)); 249} 250 251static inline int 252rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc) 253{ 254 struct ibv_cq *cq; 255 void *context; 256 int ret; 257 258 do { 259 ret = ibv_poll_cq(id->send_cq, 1, wc); 260 if (ret) 261 break; 262 263 ret = ibv_req_notify_cq(id->send_cq, 0); 264 if (ret) 265 return rdma_seterrno(ret); 266 267 ret = ibv_poll_cq(id->send_cq, 1, wc); 268 if (ret) 269 break; 270 271 ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context); 272 if (ret) 273 return ret; 274 275 assert(cq == id->send_cq && context == id); 276 ibv_ack_cq_events(id->send_cq, 1); 277 } while (1); 278 279 return (ret < 0) ? rdma_seterrno(ret) : ret; 280} 281 282static inline int 283rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc) 284{ 285 struct ibv_cq *cq; 286 void *context; 287 int ret; 288 289 do { 290 ret = ibv_poll_cq(id->recv_cq, 1, wc); 291 if (ret) 292 break; 293 294 ret = ibv_req_notify_cq(id->recv_cq, 0); 295 if (ret) 296 return rdma_seterrno(ret); 297 298 ret = ibv_poll_cq(id->recv_cq, 1, wc); 299 if (ret) 300 break; 301 302 ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context); 303 if (ret) 304 return ret; 305 306 assert(cq == id->recv_cq && context == id); 307 ibv_ack_cq_events(id->recv_cq, 1); 308 } while (1); 309 310 return (ret < 0) ? rdma_seterrno(ret) : ret; 311} 312 313#ifdef __cplusplus 314} 315#endif 316 317#endif /* RDMA_CMA_H */ 318