1/*-
2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_ib/mlx5_ib_srq.c 322810 2017-08-23 12:09:37Z hselasky $
26 */
27
28#include <linux/module.h>
29#include <dev/mlx5/qp.h>
30#include <dev/mlx5/srq.h>
31#include <linux/slab.h>
32#include <rdma/ib_umem.h>
33#include <rdma/ib_user_verbs.h>
34
35#include "mlx5_ib.h"
36#include "user.h"
37
38/* not supported currently */
39static int srq_signature;
40
41static void *get_wqe(struct mlx5_ib_srq *srq, int n)
42{
43	return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
44}
45
46static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, int type)
47{
48	struct ib_event event;
49	struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
50
51	if (ibsrq->event_handler) {
52		event.device      = ibsrq->device;
53		event.element.srq = ibsrq;
54		switch (type) {
55		case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
56			event.event = IB_EVENT_SRQ_LIMIT_REACHED;
57			break;
58		case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
59			event.event = IB_EVENT_SRQ_ERR;
60			break;
61		default:
62			printf("mlx5_ib: WARN: ""mlx5_ib: Unexpected event type %d on SRQ %06x\n", type, srq->srqn);
63			return;
64		}
65
66		ibsrq->event_handler(&event, ibsrq->srq_context);
67	}
68}
69
70static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
71			   struct mlx5_create_srq_mbox_in **in,
72			   struct ib_udata *udata, int buf_size, int *inlen)
73{
74	struct mlx5_ib_dev *dev = to_mdev(pd->device);
75	struct mlx5_ib_create_srq ucmd;
76	size_t ucmdlen;
77	void *xsrqc;
78	int err;
79	int npages;
80	int page_shift;
81	int ncont;
82	int drv_data = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
83	u32 offset;
84
85	ucmdlen = (drv_data < sizeof(ucmd)) ?
86		  drv_data : sizeof(ucmd);
87
88	if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
89		mlx5_ib_err(dev, "failed copy udata\n");
90		return -EFAULT;
91	}
92
93	if (ucmdlen == sizeof(ucmd) &&
94	    ucmd.reserved1 != 0) {
95		mlx5_ib_warn(dev, "corrupted ucmd\n");
96		return -EINVAL;
97	}
98
99	srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
100
101	srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
102				0, 0);
103	if (IS_ERR(srq->umem)) {
104		mlx5_ib_warn(dev, "failed umem get, size %d\n", buf_size);
105		err = PTR_ERR(srq->umem);
106		return err;
107	}
108
109	mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, &npages,
110			   &page_shift, &ncont, NULL);
111	err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift,
112				     &offset);
113	if (err) {
114		mlx5_ib_warn(dev, "bad offset\n");
115		goto err_umem;
116	}
117
118	*inlen = sizeof(**in) + sizeof(*(*in)->pas) * ncont;
119	*in = mlx5_vzalloc(*inlen);
120	if (!(*in)) {
121		mlx5_ib_err(dev, "failed allocate mbox\n");
122		err = -ENOMEM;
123		goto err_umem;
124	}
125
126	mlx5_ib_populate_pas(dev, srq->umem, page_shift, (*in)->pas, 0);
127
128	err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context),
129				  ucmd.db_addr, &srq->db);
130	if (err) {
131		mlx5_ib_warn(dev, "map doorbell failed\n");
132		goto err_in;
133	}
134
135	(*in)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
136	(*in)->ctx.pgoff_cqn = cpu_to_be32(offset << 26);
137
138	if (MLX5_CAP_GEN(dev->mdev, cqe_version)) {
139		xsrqc = MLX5_ADDR_OF(create_xrc_srq_in, *in,
140				     xrc_srq_context_entry);
141		/* 0xffffff means we ask to work with cqe version 0 */
142		if (drv_data > offsetof(struct mlx5_ib_create_srq, uidx))
143			MLX5_SET(xrc_srqc, xsrqc, user_index, ucmd.uidx);
144		else
145			MLX5_SET(xrc_srqc, xsrqc, user_index, 0xffffff);
146	}
147
148	return 0;
149
150err_in:
151	kvfree(*in);
152
153err_umem:
154	ib_umem_release(srq->umem);
155
156	return err;
157}
158
159static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
160			     struct mlx5_create_srq_mbox_in **in, int buf_size,
161			     int *inlen)
162{
163	int err;
164	int i;
165	struct mlx5_wqe_srq_next_seg *next;
166	int page_shift;
167	void *xsrqc;
168
169	err = mlx5_db_alloc(dev->mdev, &srq->db);
170	if (err) {
171		mlx5_ib_warn(dev, "alloc dbell rec failed\n");
172		return err;
173	}
174
175	if (mlx5_buf_alloc(dev->mdev, buf_size, PAGE_SIZE * 2, &srq->buf)) {
176		mlx5_ib_err(dev, "buf alloc failed\n");
177		err = -ENOMEM;
178		goto err_db;
179	}
180	page_shift = srq->buf.page_shift;
181
182	srq->head    = 0;
183	srq->tail    = srq->msrq.max - 1;
184	srq->wqe_ctr = 0;
185
186	for (i = 0; i < srq->msrq.max; i++) {
187		next = get_wqe(srq, i);
188		next->next_wqe_index =
189			cpu_to_be16((i + 1) & (srq->msrq.max - 1));
190	}
191
192	*inlen = sizeof(**in) + sizeof(*(*in)->pas) * srq->buf.npages;
193	*in = mlx5_vzalloc(*inlen);
194	if (!*in) {
195		mlx5_ib_err(dev, "failed allocate mbox\n");
196		err = -ENOMEM;
197		goto err_buf;
198	}
199	mlx5_fill_page_array(&srq->buf, (*in)->pas);
200
201	srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL);
202	if (!srq->wrid) {
203		err = -ENOMEM;
204		goto err_in;
205	}
206	srq->wq_sig = !!srq_signature;
207
208	(*in)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
209
210	if (MLX5_CAP_GEN(dev->mdev, cqe_version)) {
211		xsrqc = MLX5_ADDR_OF(create_xrc_srq_in, *in,
212				     xrc_srq_context_entry);
213		/* 0xffffff means we ask to work with cqe version 0 */
214		MLX5_SET(xrc_srqc, xsrqc, user_index, 0xffffff);
215	}
216
217	return 0;
218
219err_in:
220	kvfree(*in);
221
222err_buf:
223	mlx5_buf_free(dev->mdev, &srq->buf);
224
225err_db:
226	mlx5_db_free(dev->mdev, &srq->db);
227	return err;
228}
229
230static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
231{
232	mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
233	ib_umem_release(srq->umem);
234}
235
236
237static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
238{
239	kfree(srq->wrid);
240	mlx5_buf_free(dev->mdev, &srq->buf);
241	mlx5_db_free(dev->mdev, &srq->db);
242}
243
244struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
245				  struct ib_srq_init_attr *init_attr,
246				  struct ib_udata *udata)
247{
248	struct mlx5_ib_dev *dev = to_mdev(pd->device);
249	struct mlx5_ib_srq *srq;
250	int desc_size;
251	int buf_size;
252	int err;
253	struct mlx5_create_srq_mbox_in *uninitialized_var(in);
254	int uninitialized_var(inlen);
255	int is_xrc;
256	u32 flgs, xrcdn;
257	__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
258
259	/* Sanity check SRQ size before proceeding */
260	if (init_attr->attr.max_wr >= max_srq_wqes) {
261		mlx5_ib_warn(dev, "max_wr %d, cap %d\n",
262			     init_attr->attr.max_wr,
263			     max_srq_wqes);
264		return ERR_PTR(-EINVAL);
265	}
266
267	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
268	if (!srq)
269		return ERR_PTR(-ENOMEM);
270
271	mutex_init(&srq->mutex);
272	spin_lock_init(&srq->lock);
273	srq->msrq.max    = roundup_pow_of_two(init_attr->attr.max_wr + 1);
274	srq->msrq.max_gs = init_attr->attr.max_sge;
275
276	desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
277		    srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
278	desc_size = roundup_pow_of_two(desc_size);
279	desc_size = max_t(int, 32, desc_size);
280	srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
281		sizeof(struct mlx5_wqe_data_seg);
282	srq->msrq.wqe_shift = ilog2(desc_size);
283	buf_size = srq->msrq.max * desc_size;
284	mlx5_ib_dbg(dev, "desc_size 0x%x, req wr 0x%x, srq size 0x%x, max_gs 0x%x, max_avail_gather 0x%x\n",
285		    desc_size, init_attr->attr.max_wr, srq->msrq.max, srq->msrq.max_gs,
286		    srq->msrq.max_avail_gather);
287
288	if (pd->uobject)
289		err = create_srq_user(pd, srq, &in, udata, buf_size, &inlen);
290	else
291		err = create_srq_kernel(dev, srq, &in, buf_size, &inlen);
292
293	if (err) {
294		mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
295			     pd->uobject ? "user" : "kernel", err);
296		goto err_srq;
297	}
298
299	is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
300	in->ctx.state_log_sz = ilog2(srq->msrq.max);
301	flgs = ((srq->msrq.wqe_shift - 4) | (is_xrc << 5) | (srq->wq_sig << 7)) << 24;
302	xrcdn = 0;
303	if (is_xrc) {
304		xrcdn = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
305		in->ctx.pgoff_cqn |= cpu_to_be32(to_mcq(init_attr->ext.xrc.cq)->mcq.cqn);
306	} else if (init_attr->srq_type == IB_SRQT_BASIC) {
307		xrcdn = to_mxrcd(dev->devr.x0)->xrcdn;
308		in->ctx.pgoff_cqn |= cpu_to_be32(to_mcq(dev->devr.c0)->mcq.cqn);
309	}
310
311	in->ctx.flags_xrcd = cpu_to_be32((flgs & 0xFF000000) | (xrcdn & 0xFFFFFF));
312
313	in->ctx.pd = cpu_to_be32(to_mpd(pd)->pdn);
314	in->ctx.db_record = cpu_to_be64(srq->db.dma);
315	err = mlx5_core_create_srq(dev->mdev, &srq->msrq, in, inlen, is_xrc);
316	kvfree(in);
317	if (err) {
318		mlx5_ib_warn(dev, "create SRQ failed, err %d\n", err);
319		goto err_usr_kern_srq;
320	}
321
322	mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn);
323
324	srq->msrq.event = mlx5_ib_srq_event;
325	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
326
327	if (pd->uobject)
328		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) {
329			mlx5_ib_err(dev, "copy to user failed\n");
330			err = -EFAULT;
331			goto err_core;
332		}
333
334	init_attr->attr.max_wr = srq->msrq.max - 1;
335
336	return &srq->ibsrq;
337
338err_core:
339	mlx5_core_destroy_srq(dev->mdev, &srq->msrq);
340
341err_usr_kern_srq:
342	if (pd->uobject)
343		destroy_srq_user(pd, srq);
344	else
345		destroy_srq_kernel(dev, srq);
346
347err_srq:
348	kfree(srq);
349
350	return ERR_PTR(err);
351}
352
353int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
354		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
355{
356	struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
357	struct mlx5_ib_srq *srq = to_msrq(ibsrq);
358	int ret;
359
360	/* We don't support resizing SRQs yet */
361	if (attr_mask & IB_SRQ_MAX_WR)
362		return -EINVAL;
363
364	if (attr_mask & IB_SRQ_LIMIT) {
365		if (attr->srq_limit >= srq->msrq.max)
366			return -EINVAL;
367
368		mutex_lock(&srq->mutex);
369		ret = mlx5_core_arm_srq(dev->mdev, &srq->msrq, attr->srq_limit, 1);
370		mutex_unlock(&srq->mutex);
371
372		if (ret)
373			return ret;
374	}
375
376	return 0;
377}
378
379int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
380{
381	struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
382	struct mlx5_ib_srq *srq = to_msrq(ibsrq);
383	int ret;
384	struct mlx5_query_srq_mbox_out *out;
385
386	out = kzalloc(sizeof(*out), GFP_KERNEL);
387	if (!out)
388		return -ENOMEM;
389
390	ret = mlx5_core_query_srq(dev->mdev, &srq->msrq, out);
391	if (ret)
392		goto out_box;
393
394	srq_attr->srq_limit = be16_to_cpu(out->ctx.lwm);
395	srq_attr->max_wr    = srq->msrq.max - 1;
396	srq_attr->max_sge   = srq->msrq.max_gs;
397
398out_box:
399	kfree(out);
400	return ret;
401}
402
403int mlx5_ib_destroy_srq(struct ib_srq *srq)
404{
405	struct mlx5_ib_dev *dev = to_mdev(srq->device);
406	struct mlx5_ib_srq *msrq = to_msrq(srq);
407
408	mlx5_core_destroy_srq(dev->mdev, &msrq->msrq);
409
410	if (srq->uobject) {
411		mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
412		ib_umem_release(msrq->umem);
413	} else {
414		destroy_srq_kernel(dev, msrq);
415	}
416
417	kfree(srq);
418	return 0;
419}
420
421void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)
422{
423	struct mlx5_wqe_srq_next_seg *next;
424
425	/* always called with interrupts disabled. */
426	spin_lock(&srq->lock);
427
428	next = get_wqe(srq, srq->tail);
429	next->next_wqe_index = cpu_to_be16(wqe_index);
430	srq->tail = wqe_index;
431
432	spin_unlock(&srq->lock);
433}
434
435int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
436			  struct ib_recv_wr **bad_wr)
437{
438	struct mlx5_ib_srq *srq = to_msrq(ibsrq);
439	struct mlx5_wqe_srq_next_seg *next;
440	struct mlx5_wqe_data_seg *scat;
441	struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
442	struct mlx5_core_dev *mdev = dev->mdev;
443	unsigned long flags;
444	int err = 0;
445	int nreq;
446	int i;
447
448	spin_lock_irqsave(&srq->lock, flags);
449
450	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
451		err = -EIO;
452		*bad_wr = wr;
453		nreq = 0;
454		goto out;
455	}
456
457	for (nreq = 0; wr; nreq++, wr = wr->next) {
458		if (unlikely(wr->num_sge > srq->msrq.max_gs)) {
459			err = -EINVAL;
460			*bad_wr = wr;
461			break;
462		}
463
464		if (unlikely(srq->head == srq->tail)) {
465			err = -ENOMEM;
466			*bad_wr = wr;
467			break;
468		}
469
470		srq->wrid[srq->head] = wr->wr_id;
471
472		next      = get_wqe(srq, srq->head);
473		srq->head = be16_to_cpu(next->next_wqe_index);
474		scat      = (struct mlx5_wqe_data_seg *)(next + 1);
475
476		for (i = 0; i < wr->num_sge; i++) {
477			scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length);
478			scat[i].lkey       = cpu_to_be32(wr->sg_list[i].lkey);
479			scat[i].addr       = cpu_to_be64(wr->sg_list[i].addr);
480		}
481
482		if (i < srq->msrq.max_avail_gather) {
483			scat[i].byte_count = 0;
484			scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
485			scat[i].addr       = 0;
486		}
487	}
488
489	if (likely(nreq)) {
490		srq->wqe_ctr += nreq;
491
492		/* Make sure that descriptors are written before
493		 * doorbell record.
494		 */
495		wmb();
496
497		*srq->db.db = cpu_to_be32(srq->wqe_ctr);
498	}
499out:
500	spin_unlock_irqrestore(&srq->lock, flags);
501
502	return err;
503}
504