1/*
2 * Copyright (c) 2010-2014 Intel Corporation.  All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *        copyright notice, this list of conditions and the following
16 *        disclaimer.
17 *
18 *      - Redistributions in binary form must reproduce the above
19 *        copyright notice, this list of conditions and the following
20 *        disclaimer in the documentation and/or other materials
21 *        provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#if !defined(RDMA_VERBS_H)
34#define RDMA_VERBS_H
35
36#include <assert.h>
37#include <infiniband/verbs.h>
38#include <rdma/rdma_cma.h>
39#include <errno.h>
40
41#ifdef __cplusplus
42extern "C" {
43#endif
44
45static inline int rdma_seterrno(int ret)
46{
47	if (ret) {
48		errno = ret;
49		ret = -1;
50	}
51	return ret;
52}
53
54/*
55 * Shared receive queues.
56 */
57int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd,
58		    struct ibv_srq_init_attr *attr);
59int rdma_create_srq_ex(struct rdma_cm_id *id, struct ibv_srq_init_attr_ex *attr);
60
61void rdma_destroy_srq(struct rdma_cm_id *id);
62
63
64/*
65 * Memory registration helpers.
66 */
67static inline struct ibv_mr *
68rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
69{
70	return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
71}
72
73static inline struct ibv_mr *
74rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
75{
76	return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
77						IBV_ACCESS_REMOTE_READ);
78}
79
80static inline struct ibv_mr *
81rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
82{
83	return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
84						IBV_ACCESS_REMOTE_WRITE);
85}
86
87static inline int
88rdma_dereg_mr(struct ibv_mr *mr)
89{
90	return rdma_seterrno(ibv_dereg_mr(mr));
91}
92
93
94/*
95 * Vectored send, receive, and RDMA operations.
96 * Support multiple scatter-gather entries.
97 */
98static inline int
99rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
100		int nsge)
101{
102	struct ibv_recv_wr wr, *bad;
103
104	wr.wr_id = (uintptr_t) context;
105	wr.next = NULL;
106	wr.sg_list = sgl;
107	wr.num_sge = nsge;
108
109	if (id->srq)
110		return rdma_seterrno(ibv_post_srq_recv(id->srq, &wr, &bad));
111	else
112		return rdma_seterrno(ibv_post_recv(id->qp, &wr, &bad));
113}
114
115static inline int
116rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
117		int nsge, int flags)
118{
119	struct ibv_send_wr wr, *bad;
120
121	wr.wr_id = (uintptr_t) context;
122	wr.next = NULL;
123	wr.sg_list = sgl;
124	wr.num_sge = nsge;
125	wr.opcode = IBV_WR_SEND;
126	wr.send_flags = flags;
127
128	return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
129}
130
131static inline int
132rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
133		int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
134{
135	struct ibv_send_wr wr, *bad;
136
137	wr.wr_id = (uintptr_t) context;
138	wr.next = NULL;
139	wr.sg_list = sgl;
140	wr.num_sge = nsge;
141	wr.opcode = IBV_WR_RDMA_READ;
142	wr.send_flags = flags;
143	wr.wr.rdma.remote_addr = remote_addr;
144	wr.wr.rdma.rkey = rkey;
145
146	return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
147}
148
149static inline int
150rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
151		 int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
152{
153	struct ibv_send_wr wr, *bad;
154
155	wr.wr_id = (uintptr_t) context;
156	wr.next = NULL;
157	wr.sg_list = sgl;
158	wr.num_sge = nsge;
159	wr.opcode = IBV_WR_RDMA_WRITE;
160	wr.send_flags = flags;
161	wr.wr.rdma.remote_addr = remote_addr;
162	wr.wr.rdma.rkey = rkey;
163
164	return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
165}
166
167/*
168 * Simple send, receive, and RDMA calls.
169 */
170static inline int
171rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
172	       size_t length, struct ibv_mr *mr)
173{
174	struct ibv_sge sge;
175
176	assert((addr >= mr->addr) &&
177		(((uint8_t *) addr + length) <= ((uint8_t *) mr->addr + mr->length)));
178	sge.addr = (uint64_t) (uintptr_t) addr;
179	sge.length = (uint32_t) length;
180	sge.lkey = mr->lkey;
181
182	return rdma_post_recvv(id, context, &sge, 1);
183}
184
185static inline int
186rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
187	       size_t length, struct ibv_mr *mr, int flags)
188{
189	struct ibv_sge sge;
190
191	sge.addr = (uint64_t) (uintptr_t) addr;
192	sge.length = (uint32_t) length;
193	sge.lkey = mr ? mr->lkey : 0;
194
195	return rdma_post_sendv(id, context, &sge, 1, flags);
196}
197
198static inline int
199rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
200	       size_t length, struct ibv_mr *mr, int flags,
201	       uint64_t remote_addr, uint32_t rkey)
202{
203	struct ibv_sge sge;
204
205	sge.addr = (uint64_t) (uintptr_t) addr;
206	sge.length = (uint32_t) length;
207	sge.lkey = mr->lkey;
208
209	return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
210}
211
212static inline int
213rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
214		size_t length, struct ibv_mr *mr, int flags,
215		uint64_t remote_addr, uint32_t rkey)
216{
217	struct ibv_sge sge;
218
219	sge.addr = (uint64_t) (uintptr_t) addr;
220	sge.length = (uint32_t) length;
221	sge.lkey = mr ? mr->lkey : 0;
222
223	return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
224}
225
226static inline int
227rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
228		  size_t length, struct ibv_mr *mr, int flags,
229		  struct ibv_ah *ah, uint32_t remote_qpn)
230{
231	struct ibv_send_wr wr, *bad;
232	struct ibv_sge sge;
233
234	sge.addr = (uint64_t) (uintptr_t) addr;
235	sge.length = (uint32_t) length;
236	sge.lkey = mr ? mr->lkey : 0;
237
238	wr.wr_id = (uintptr_t) context;
239	wr.next = NULL;
240	wr.sg_list = &sge;
241	wr.num_sge = 1;
242	wr.opcode = IBV_WR_SEND;
243	wr.send_flags = flags;
244	wr.wr.ud.ah = ah;
245	wr.wr.ud.remote_qpn = remote_qpn;
246	wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
247
248	return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
249}
250
251static inline int
252rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
253{
254	struct ibv_cq *cq;
255	void *context;
256	int ret;
257
258	do {
259		ret = ibv_poll_cq(id->send_cq, 1, wc);
260		if (ret)
261			break;
262
263		ret = ibv_req_notify_cq(id->send_cq, 0);
264		if (ret)
265			return rdma_seterrno(ret);
266
267		ret = ibv_poll_cq(id->send_cq, 1, wc);
268		if (ret)
269			break;
270
271		ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
272		if (ret)
273			return ret;
274
275		assert(cq == id->send_cq && context == id);
276		ibv_ack_cq_events(id->send_cq, 1);
277	} while (1);
278
279	return (ret < 0) ? rdma_seterrno(ret) : ret;
280}
281
282static inline int
283rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
284{
285	struct ibv_cq *cq;
286	void *context;
287	int ret;
288
289	do {
290		ret = ibv_poll_cq(id->recv_cq, 1, wc);
291		if (ret)
292			break;
293
294		ret = ibv_req_notify_cq(id->recv_cq, 0);
295		if (ret)
296			return rdma_seterrno(ret);
297
298		ret = ibv_poll_cq(id->recv_cq, 1, wc);
299		if (ret)
300			break;
301
302		ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context);
303		if (ret)
304			return ret;
305
306		assert(cq == id->recv_cq && context == id);
307		ibv_ack_cq_events(id->recv_cq, 1);
308	} while (1);
309
310	return (ret < 0) ? rdma_seterrno(ret) : ret;
311}
312
313#ifdef __cplusplus
314}
315#endif
316
317#endif /* RDMA_CMA_H */
318