mlx5_ib.h revision 323223
1251881Speter/*-
2251881Speter * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
3251881Speter *
4251881Speter * Redistribution and use in source and binary forms, with or without
5251881Speter * modification, are permitted provided that the following conditions
6251881Speter * are met:
7251881Speter * 1. Redistributions of source code must retain the above copyright
8251881Speter *    notice, this list of conditions and the following disclaimer.
9251881Speter * 2. Redistributions in binary form must reproduce the above copyright
10251881Speter *    notice, this list of conditions and the following disclaimer in the
11251881Speter *    documentation and/or other materials provided with the distribution.
12251881Speter *
13251881Speter * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14251881Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15251881Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16251881Speter * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17251881Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18251881Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19251881Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20251881Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21251881Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22251881Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23251881Speter * SUCH DAMAGE.
24251881Speter *
25251881Speter * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_ib/mlx5_ib.h 323223 2017-09-06 15:33:23Z hselasky $
26251881Speter */
27251881Speter
28251881Speter#ifndef MLX5_IB_H
29251881Speter#define MLX5_IB_H
30251881Speter
31251881Speter#include <linux/kernel.h>
32251881Speter#include <linux/sched.h>
33251881Speter#include <rdma/ib_verbs.h>
34251881Speter#include <rdma/ib_smi.h>
35251881Speter#include <rdma/ib_addr.h>
36251881Speter#include <dev/mlx5/device.h>
37251881Speter#include <dev/mlx5/driver.h>
38251881Speter#include <dev/mlx5/cq.h>
39251881Speter#include <dev/mlx5/qp.h>
40251881Speter#include <dev/mlx5/srq.h>
41251881Speter#include <linux/types.h>
42251881Speter#include <dev/mlx5/mlx5_core/transobj.h>
43251881Speter
44251881Speter#define mlx5_ib_dbg(dev, format, arg...)				\
45251881Speterpr_debug("mlx5_dbg:%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__,	\
46251881Speter	 __LINE__, curthread->td_proc->p_pid, ##arg)
47251881Speter
48251881Speter#define mlx5_ib_err(dev, format, arg...)				\
49251881Speterprintf("mlx5_ib: ERR: ""mlx5_err:%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
50251881Speter	__LINE__, curthread->td_proc->p_pid, ##arg)
51251881Speter
52251881Speter#define mlx5_ib_warn(dev, format, arg...)				\
53251881Speterprintf("mlx5_ib: WARN: ""mlx5_warn:%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
54251881Speter	__LINE__, curthread->td_proc->p_pid, ##arg)
55251881Speter#define BF_ENABLE 0
56251881Speter
57251881Speterextern struct workqueue_struct *mlx5_ib_wq;
58251881Speter
59251881Speterenum {
60251881Speter	MLX5_IB_MMAP_CMD_SHIFT	= 8,
61251881Speter	MLX5_IB_MMAP_CMD_MASK	= 0xff,
62251881Speter};
63251881Speter
64251881Speterenum mlx5_ib_mmap_cmd {
65251881Speter	MLX5_IB_MMAP_REGULAR_PAGE		= 0,
66251881Speter	MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES	= 1,
67251881Speter	MLX5_IB_MMAP_WC_PAGE			= 2,
68251881Speter	MLX5_IB_MMAP_NC_PAGE			= 3,
69251881Speter	MLX5_IB_MMAP_MAP_DC_INFO_PAGE		= 4,
70251881Speter
71251881Speter	/* Use EXP mmap commands until it is pushed to upstream */
72251881Speter	MLX5_IB_EXP_MMAP_CORE_CLOCK			= 0xFB,
73251881Speter	MLX5_IB_EXP_MMAP_GET_CONTIGUOUS_PAGES_CPU_NUMA	= 0xFC,
74251881Speter	MLX5_IB_EXP_MMAP_GET_CONTIGUOUS_PAGES_DEV_NUMA	= 0xFD,
75251881Speter	MLX5_IB_EXP_ALLOC_N_MMAP_WC			= 0xFE,
76251881Speter};
77251881Speter
78251881Speterenum {
79251881Speter	MLX5_RES_SCAT_DATA32_CQE	= 0x1,
80251881Speter	MLX5_RES_SCAT_DATA64_CQE	= 0x2,
81251881Speter	MLX5_REQ_SCAT_DATA32_CQE	= 0x11,
82251881Speter	MLX5_REQ_SCAT_DATA64_CQE	= 0x22,
83251881Speter};
84251881Speter
85251881Speterenum {
86251881Speter	MLX5_DCT_CS_RES_64		= 2,
87251881Speter	MLX5_CNAK_RX_POLL_CQ_QUOTA	= 256,
88251881Speter};
89251881Speter
90251881Speterenum mlx5_ib_latency_class {
91251881Speter	MLX5_IB_LATENCY_CLASS_LOW,
92251881Speter	MLX5_IB_LATENCY_CLASS_MEDIUM,
93251881Speter	MLX5_IB_LATENCY_CLASS_HIGH,
94251881Speter	MLX5_IB_LATENCY_CLASS_FAST_PATH
95251881Speter};
96251881Speter
97251881Speterenum mlx5_ib_mad_ifc_flags {
98251881Speter	MLX5_MAD_IFC_IGNORE_MKEY	= 1,
99251881Speter	MLX5_MAD_IFC_IGNORE_BKEY	= 2,
100251881Speter	MLX5_MAD_IFC_NET_VIEW		= 4,
101251881Speter};
102251881Speter
103251881Speterenum {
104251881Speter	MLX5_CROSS_CHANNEL_UUAR		= 0,
105251881Speter};
106251881Speter
107251881Speterenum {
108251881Speter	MLX5_IB_MAX_CTX_DYNAMIC_UARS = 256,
109251881Speter	MLX5_IB_INVALID_UAR_INDEX = -1U
110251881Speter};
111251881Speter
112251881Speterenum {
113251881Speter	MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES	= 13,
114251881Speter	MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES	= 6,
115251881Speter	MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES	= 16,
116251881Speter	MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES	= 9,
117251881Speter};
118251881Speter
119251881Speterstruct mlx5_ib_ucontext {
120251881Speter	struct ib_ucontext	ibucontext;
121251881Speter	struct list_head	db_page_list;
122251881Speter
123251881Speter	/* protect doorbell record alloc/free
124251881Speter	 */
125251881Speter	struct mutex		db_page_mutex;
126251881Speter	struct mlx5_uuar_info	uuari;
127251881Speter	u32			dynamic_wc_uar_index[MLX5_IB_MAX_CTX_DYNAMIC_UARS];
128251881Speter	/* Transport Domain number */
129251881Speter	u32			tdn;
130251881Speter};
131251881Speter
132251881Speterstatic inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
133251881Speter{
134251881Speter	return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
135251881Speter}
136251881Speter
137251881Speterstruct mlx5_ib_pd {
138251881Speter	struct ib_pd		ibpd;
139251881Speter	u32			pdn;
140251881Speter	u32			pa_lkey;
141251881Speter};
142251881Speter
143251881Speterstruct wr_list {
144251881Speter	u16	opcode;
145251881Speter	u16	next;
146251881Speter};
147251881Speter
148251881Speterstruct mlx5_swr_ctx {
149251881Speter	u64		wrid;
150251881Speter	u32		wr_data;
151251881Speter	struct wr_list	w_list;
152251881Speter	u32		wqe_head;
153251881Speter	u8		sig_piped;
154251881Speter	u8		rsvd[11];
155251881Speter};
156251881Speter
157251881Speterstruct mlx5_rwr_ctx {
158251881Speter	u64		       wrid;
159251881Speter};
160251881Speter
161251881Speterstruct mlx5_ib_wq {
162251881Speter	union {
163251881Speter		struct mlx5_swr_ctx *swr_ctx;
164251881Speter		struct mlx5_rwr_ctx *rwr_ctx;
165251881Speter	};
166251881Speter	u16		        unsig_count;
167251881Speter
168251881Speter	/* serialize post to the work queue
169251881Speter	 */
170251881Speter	spinlock_t		lock;
171251881Speter	int			wqe_cnt;
172251881Speter	int			max_post;
173251881Speter	int			max_gs;
174251881Speter	int			offset;
175251881Speter	int			wqe_shift;
176251881Speter	unsigned		head;
177251881Speter	unsigned		tail;
178251881Speter	u16			cur_post;
179251881Speter	u16			last_poll;
180251881Speter	void		       *qend;
181251881Speter};
182251881Speter
183251881Speterenum {
184251881Speter	MLX5_QP_USER,
185251881Speter	MLX5_QP_KERNEL,
186251881Speter	MLX5_QP_EMPTY
187251881Speter};
188251881Speter
189251881Speterenum {
190251881Speter	MLX5_WQ_USER,
191251881Speter	MLX5_WQ_KERNEL
192251881Speter};
193251881Speter
194251881Speterstruct mlx5_ib_sqd {
195251881Speter	struct mlx5_ib_qp	*qp;
196251881Speter	struct work_struct	work;
197251881Speter};
198251881Speter
199251881Speterstruct mlx5_ib_mc_flows_list {
200251881Speter	struct list_head		flows_list;
201251881Speter	/*Protect the flows_list*/
202251881Speter	struct mutex		lock;
203251881Speter};
204251881Speter
205251881Speterstruct mlx5_ib_qp {
206251881Speter	struct ib_qp		ibqp;
207251881Speter	struct mlx5_core_qp	mqp;
208251881Speter	struct mlx5_core_qp	mrq;
209251881Speter	struct mlx5_core_qp	msq;
210251881Speter	u32			tisn;
211251881Speter	u32			tirn;
212251881Speter	struct mlx5_buf		buf;
213251881Speter
214251881Speter	struct mlx5_db		db;
215251881Speter	struct mlx5_ib_wq	rq;
216251881Speter
217251881Speter	u32			doorbell_qpn;
218251881Speter	u8			sq_signal_bits;
219251881Speter	u8			fm_cache;
220251881Speter	int			sq_max_wqes_per_wr;
221251881Speter	int			sq_spare_wqes;
222251881Speter	struct mlx5_ib_wq	sq;
223251881Speter
224251881Speter	struct ib_umem	       *umem;
225251881Speter	int			buf_size;
226251881Speter	/* Raw Ethernet QP's SQ is allocated seperately
227251881Speter	 * from the RQ's buffer in user-space.
228251881Speter	 */
229251881Speter	struct ib_umem	       *sq_umem;
230251881Speter	int			sq_buf_size;
231251881Speter	u64			sq_buf_addr;
232251881Speter	int			allow_mp_wqe;
233251881Speter
234251881Speter	/* serialize qp state modifications
235251881Speter	 */
236251881Speter	struct mutex		mutex;
237251881Speter	u16			xrcdn;
238251881Speter	u32			flags;
239251881Speter	u8			port;
240251881Speter	u8			alt_port;
241251881Speter	u8			atomic_rd_en;
242251881Speter	u8			resp_depth;
243251881Speter	u8			state;
244251881Speter	/* Raw Ethernet QP's SQ and RQ states */
245251881Speter	u8			rq_state;
246251881Speter	u8			sq_state;
247251881Speter	int			mlx_type;
248251881Speter	int			wq_sig;
249251881Speter	int			scat_cqe;
250251881Speter	int			max_inline_data;
251251881Speter	struct mlx5_bf	       *bf;
252251881Speter	int			has_rq;
253251881Speter
254251881Speter	/* only for user space QPs. For kernel
255251881Speter	 * we have it from the bf object
256251881Speter	 */
257251881Speter	int			uuarn;
258251881Speter
259251881Speter	int			create_type;
260251881Speter	u32			pa_lkey;
261251881Speter
262251881Speter	/* Store signature errors */
263251881Speter	bool			signature_en;
264251881Speter
265251881Speter	struct list_head	qps_list;
266251881Speter	struct list_head	cq_recv_list;
267251881Speter	struct list_head	cq_send_list;
268251881Speter
269251881Speter	struct mlx5_ib_mc_flows_list mc_flows_list;
270251881Speter};
271251881Speter
272251881Speterstruct mlx5_ib_cq_buf {
273251881Speter	struct mlx5_buf		buf;
274251881Speter	struct ib_umem		*umem;
275251881Speter	int			cqe_size;
276251881Speter	int			nent;
277251881Speter};
278251881Speter
279251881Speterenum mlx5_ib_qp_flags {
280251881Speter	MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK     = 1 << 0,
281251881Speter	MLX5_IB_QP_SIGNATURE_HANDLING           = 1 << 1,
282251881Speter	MLX5_IB_QP_CAP_RX_END_PADDING		= 1 << 5,
283251881Speter};
284251881Speter
285251881Speterstruct mlx5_umr_wr {
286251881Speter	union {
287251881Speter		u64			virt_addr;
288251881Speter		u64			offset;
289251881Speter	} target;
290251881Speter	struct ib_pd		       *pd;
291251881Speter	unsigned int			page_shift;
292251881Speter	unsigned int			npages;
293251881Speter	u64				length;
294251881Speter	int				access_flags;
295251881Speter	u32				mkey;
296251881Speter};
297251881Speter
298251881Speterstruct mlx5_shared_mr_info {
299251881Speter	int mr_id;
300251881Speter	struct ib_umem		*umem;
301251881Speter};
302251881Speter
303251881Speterstruct mlx5_ib_cq {
304251881Speter	struct ib_cq		ibcq;
305251881Speter	struct mlx5_core_cq	mcq;
306251881Speter	struct mlx5_ib_cq_buf	buf;
307251881Speter	struct mlx5_db		db;
308251881Speter
309251881Speter	/* serialize access to the CQ
310251881Speter	 */
311251881Speter	spinlock_t		lock;
312251881Speter
313251881Speter	/* protect resize cq
314251881Speter	 */
315251881Speter	struct mutex		resize_mutex;
316251881Speter	struct mlx5_ib_cq_buf  *resize_buf;
317251881Speter	struct ib_umem	       *resize_umem;
318251881Speter	int			cqe_size;
319251881Speter	struct list_head		list_send_qp;
320251881Speter	struct list_head		list_recv_qp;
321251881Speter};
322251881Speter
323251881Speterstruct mlx5_ib_srq {
324251881Speter	struct ib_srq		ibsrq;
325251881Speter	struct mlx5_core_srq	msrq;
326251881Speter	struct mlx5_buf		buf;
327251881Speter	struct mlx5_db		db;
328251881Speter	u64		       *wrid;
329251881Speter	/* protect SRQ hanlding
330251881Speter	 */
331251881Speter	spinlock_t		lock;
332251881Speter	int			head;
333251881Speter	int			tail;
334251881Speter	u16			wqe_ctr;
335251881Speter	struct ib_umem	       *umem;
336251881Speter	/* serialize arming a SRQ
337251881Speter	 */
338251881Speter	struct mutex		mutex;
339251881Speter	int			wq_sig;
340251881Speter};
341251881Speter
342251881Speterstruct mlx5_ib_xrcd {
343251881Speter	struct ib_xrcd		ibxrcd;
344251881Speter	u32			xrcdn;
345251881Speter};
346251881Speter
347251881Speterenum mlx5_ib_mtt_access_flags {
348251881Speter	MLX5_IB_MTT_READ  = (1 << 0),
349251881Speter	MLX5_IB_MTT_WRITE = (1 << 1),
350251881Speter};
351251881Speter
352251881Speter#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
353251881Speter
354251881Speterstruct mlx5_ib_mr {
355251881Speter	struct ib_mr		ibmr;
356251881Speter	struct mlx5_core_mr	mmr;
357251881Speter	struct ib_umem	       *umem;
358251881Speter	struct mlx5_shared_mr_info	*smr_info;
359251881Speter	struct list_head	list;
360251881Speter	int			order;
361251881Speter	int			umred;
362251881Speter	dma_addr_t		dma;
363251881Speter	int			npages;
364251881Speter	struct mlx5_ib_dev     *dev;
365251881Speter	struct mlx5_create_mkey_mbox_out out;
366251881Speter	struct mlx5_core_sig_ctx    *sig;
367251881Speter	u32			max_reg_descriptors;
368251881Speter	u64			size;
369251881Speter	u64			page_count;
370251881Speter	struct mlx5_ib_mr     **children;
371251881Speter	int			nchild;
372251881Speter};
373251881Speter
374251881Speterstruct mlx5_ib_fast_reg_page_list {
375251881Speter	struct ib_fast_reg_page_list	ibfrpl;
376251881Speter	__be64			       *mapped_page_list;
377251881Speter	dma_addr_t			map;
378251881Speter};
379251881Speter
380251881Speterstruct mlx5_ib_umr_context {
381251881Speter	enum ib_wc_status	status;
382251881Speter	struct completion	done;
383251881Speter};
384251881Speter
385251881Speterstatic inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
386251881Speter{
387251881Speter	context->status = -1;
388251881Speter	init_completion(&context->done);
389251881Speter}
390251881Speter
391251881Speterstruct umr_common {
392251881Speter	struct ib_pd	*pd;
393251881Speter	struct ib_mr	*mr;
394251881Speter};
395251881Speter
396251881Speterenum {
397251881Speter	MLX5_FMR_INVALID,
398251881Speter	MLX5_FMR_VALID,
399251881Speter	MLX5_FMR_BUSY,
400251881Speter};
401251881Speter
402251881Speterstruct mlx5_ib_fmr {
403251881Speter	struct ib_fmr			ibfmr;
404251881Speter	struct mlx5_core_mr		mr;
405251881Speter	int				access_flags;
406251881Speter	int				state;
407251881Speter	/* protect fmr state
408251881Speter	 */
409251881Speter	spinlock_t			lock;
410251881Speter	u64				wrid;
411251881Speter	struct ib_send_wr		wr[2];
412251881Speter	u8				page_shift;
413251881Speter	struct ib_fast_reg_page_list	page_list;
414251881Speter};
415251881Speter
416251881Speterstruct cache_order {
417251881Speter	struct kobject		kobj;
418251881Speter	int			order;
419251881Speter	int			index;
420251881Speter	struct mlx5_ib_dev     *dev;
421251881Speter};
422251881Speter
423251881Speterstruct mlx5_cache_ent {
424251881Speter	struct list_head	head;
425251881Speter	/* sync access to the cahce entry
426251881Speter	 */
427251881Speter	spinlock_t		lock;
428251881Speter
429251881Speter
430251881Speter	u32                     order;
431	u32			size;
432	u32                     cur;
433	u32                     miss;
434	u32			limit;
435
436	struct mlx5_ib_dev     *dev;
437	struct work_struct	work;
438	struct delayed_work	dwork;
439	int			pending;
440	struct cache_order	co;
441};
442
443struct mlx5_mr_cache {
444	struct workqueue_struct *wq;
445	struct mlx5_cache_ent	ent[MAX_MR_CACHE_ENTRIES];
446	int			stopped;
447	struct dentry		*root;
448	int		last_add;
449	int			rel_timeout;
450	int			rel_imm;
451};
452
453struct mlx5_ib_resources {
454	struct ib_cq	*c0;
455	struct ib_xrcd	*x0;
456	struct ib_xrcd	*x1;
457	struct ib_pd	*p0;
458	struct ib_srq	*s0;
459	struct ib_srq	*s1;
460};
461
462struct mlx5_dc_tracer {
463	struct page	*pg;
464	dma_addr_t	dma;
465	int		size;
466	int		order;
467};
468
469struct mlx5_dc_desc {
470	dma_addr_t	dma;
471	void		*buf;
472};
473
474enum mlx5_op {
475	MLX5_WR_OP_MLX	= 1,
476};
477
478struct mlx5_mlx_wr {
479	u8	sl;
480	u16	dlid;
481	int	icrc;
482};
483
484struct mlx5_send_wr {
485	struct ib_send_wr	wr;
486	union {
487		struct mlx5_mlx_wr	mlx;
488	} sel;
489};
490
491struct mlx5_dc_data {
492	struct ib_mr		*mr;
493	struct ib_qp		*dcqp;
494	struct ib_cq		*rcq;
495	struct ib_cq		*scq;
496	unsigned int		rx_npages;
497	unsigned int		tx_npages;
498	struct mlx5_dc_desc	*rxdesc;
499	struct mlx5_dc_desc	*txdesc;
500	unsigned int		max_wqes;
501	unsigned int		cur_send;
502	unsigned int		last_send_completed;
503	int			tx_pending;
504	struct mlx5_ib_dev	*dev;
505	int			port;
506	int			initialized;
507	struct kobject		kobj;
508	unsigned long		connects;
509	unsigned long		cnaks;
510	unsigned long		discards;
511	struct ib_wc		wc_tbl[MLX5_CNAK_RX_POLL_CQ_QUOTA];
512};
513
514struct mlx5_ib_port_sysfs_group {
515	struct kobject		kobj;
516	bool   enabled;
517	struct attribute_group	counters;
518};
519
520#define	MLX5_IB_GID_MAX	16
521
522struct mlx5_ib_port {
523	struct mlx5_ib_dev	*dev;
524	u8  port_num;	/* 0 based */
525	u8  port_gone;	/* set when gone */
526	u16 q_cnt_id;
527	struct mlx5_ib_port_sysfs_group group;
528	union ib_gid gid_table[MLX5_IB_GID_MAX];
529};
530
531struct mlx5_ib_dev {
532	struct ib_device		ib_dev;
533	struct mlx5_core_dev		*mdev;
534	MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
535	int				num_ports;
536	/* serialize update of capability mask
537	 */
538	struct mutex			cap_mask_mutex;
539	bool	ib_active;
540	struct umr_common		umrc;
541	/* sync used page count stats
542	 */
543	struct mlx5_ib_resources	devr;
544	struct mutex		slow_path_mutex;
545	int				enable_atomic_resp;
546	enum ib_atomic_cap		atomic_cap;
547	struct mlx5_mr_cache		cache;
548	struct kobject               mr_cache;
549	/* protect resources needed as part of reset flow */
550	spinlock_t		reset_flow_resource_lock;
551	struct list_head		qp_list;
552	struct timer_list		delay_timer;
553	int				fill_delay;
554	struct mlx5_dc_tracer	dctr;
555	struct mlx5_dc_data	dcd[MLX5_MAX_PORTS];
556	struct kobject		*dc_kobj;
557	/* Array with num_ports elements */
558	struct mlx5_ib_port	*port;
559	struct kobject		*ports_parent;
560};
561
562static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
563{
564	return container_of(mcq, struct mlx5_ib_cq, mcq);
565}
566
567static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
568{
569	return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
570}
571
572static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
573{
574	return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
575}
576
577static inline struct mlx5_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)
578{
579	return container_of(ibfmr, struct mlx5_ib_fmr, ibfmr);
580}
581
582static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
583{
584	return container_of(ibcq, struct mlx5_ib_cq, ibcq);
585}
586
587static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
588{
589	return container_of(mqp, struct mlx5_ib_qp, mqp);
590}
591
592static inline struct mlx5_ib_qp *sq_to_mibqp(struct mlx5_core_qp *msq)
593{
594	return container_of(msq, struct mlx5_ib_qp, msq);
595}
596
597static inline struct mlx5_ib_qp *rq_to_mibqp(struct mlx5_core_qp *mrq)
598{
599	return container_of(mrq, struct mlx5_ib_qp, mrq);
600}
601
602static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmr)
603{
604	return container_of(mmr, struct mlx5_ib_mr, mmr);
605}
606
607static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
608{
609	return container_of(ibpd, struct mlx5_ib_pd, ibpd);
610}
611
612static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
613{
614	return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
615}
616
617static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
618{
619	return container_of(ibqp, struct mlx5_ib_qp, ibqp);
620}
621
622static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
623{
624	return container_of(msrq, struct mlx5_ib_srq, msrq);
625}
626
627static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
628{
629	return container_of(ibmr, struct mlx5_ib_mr, ibmr);
630}
631
632static inline struct mlx5_ib_fast_reg_page_list *to_mfrpl(struct ib_fast_reg_page_list *ibfrpl)
633{
634	return container_of(ibfrpl, struct mlx5_ib_fast_reg_page_list, ibfrpl);
635}
636
637struct mlx5_ib_ah {
638	struct ib_ah		ibah;
639	struct mlx5_av		av;
640};
641
642static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
643{
644	return container_of(ibah, struct mlx5_ib_ah, ibah);
645}
646
647int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, uintptr_t virt,
648			struct mlx5_db *db);
649void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
650void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
651void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
652void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
653int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
654		 u8 port, struct ib_wc *in_wc, struct ib_grh *in_grh,
655		 void *in_mad, void *response_mad);
656int mlx5_ib_resolve_grh(const struct ib_ah_attr *ah_attr, u8 *mac, int *is_mcast);
657struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev, struct ib_ah_attr *ah_attr,
658			   struct mlx5_ib_ah *ah, enum rdma_link_layer ll);
659struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
660int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
661int mlx5_ib_destroy_ah(struct ib_ah *ah);
662struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
663				  struct ib_srq_init_attr *init_attr,
664				  struct ib_udata *udata);
665int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
666		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
667int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
668int mlx5_ib_destroy_srq(struct ib_srq *srq);
669int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
670			  struct ib_recv_wr **bad_wr);
671struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
672				struct ib_qp_init_attr *init_attr,
673				struct ib_udata *udata);
674int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
675		      int attr_mask, struct ib_udata *udata);
676int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
677		     struct ib_qp_init_attr *qp_init_attr);
678int mlx5_ib_destroy_qp(struct ib_qp *qp);
679int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
680		      struct ib_send_wr **bad_wr);
681int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
682		      struct ib_recv_wr **bad_wr);
683void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
684struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
685				int entries, int vector,
686				struct ib_ucontext *context,
687				struct ib_udata *udata);
688int mlx5_ib_destroy_cq(struct ib_cq *cq);
689int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
690int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
691int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
692int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
693struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
694struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
695				  u64 virt_addr, int access_flags,
696				  struct ib_udata *udata, int mr_id);
697struct ib_mr *mlx5_ib_reg_phys_mr(struct ib_pd *pd,
698				  struct ib_phys_buf *buffer_list,
699				  int num_phys_buf,
700				  int access_flags,
701				  u64 *virt_addr);
702int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
703int mlx5_ib_destroy_mr(struct ib_mr *ibmr);
704struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
705					int max_page_list_len);
706struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
707							       int page_list_len);
708void mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list);
709
710struct ib_fmr *mlx5_ib_fmr_alloc(struct ib_pd *pd, int acc,
711				 struct ib_fmr_attr *fmr_attr);
712int mlx5_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
713		      int npages, u64 iova);
714int mlx5_ib_unmap_fmr(struct list_head *fmr_list);
715int mlx5_ib_fmr_dealloc(struct ib_fmr *ibfmr);
716int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
717			struct ib_wc *in_wc, struct ib_grh *in_grh,
718			struct ib_mad *in_mad, struct ib_mad *out_mad);
719struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
720					  struct ib_ucontext *context,
721					  struct ib_udata *udata);
722int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd);
723int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
724int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
725int mlx5_query_smp_attr_node_info_mad_ifc(struct ib_device *ibdev,
726					  struct ib_smp *out_mad);
727int mlx5_query_system_image_guid_mad_ifc(struct ib_device *ibdev,
728					 __be64 *sys_image_guid);
729int mlx5_query_max_pkeys_mad_ifc(struct ib_device *ibdev,
730				 u16 *max_pkeys);
731int mlx5_query_vendor_id_mad_ifc(struct ib_device *ibdev,
732				 u32 *vendor_id);
733int mlx5_query_pkey_mad_ifc(struct ib_device *ibdev, u8 port, u16 index,
734			    u16 *pkey);
735int mlx5_query_node_desc_mad_ifc(struct mlx5_ib_dev *dev, char *node_desc);
736int mlx5_query_node_guid_mad_ifc(struct mlx5_ib_dev *dev, u64 *node_guid);
737int mlx5_query_gids_mad_ifc(struct ib_device *ibdev, u8 port, int index,
738			    union ib_gid *gid);
739int mlx5_query_port_mad_ifc(struct ib_device *ibdev, u8 port,
740			    struct ib_port_attr *props);
741int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
742		       struct ib_port_attr *props);
743int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
744void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
745void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
746			int *ncont, int *order);
747void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
748			  int page_shift, __be64 *pas, int umr);
749void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
750int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
751int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
752int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
753int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
754void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context);
755int mlx5_query_port_roce(struct ib_device *ibdev, u8 port,
756			 struct ib_port_attr *props);
757__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port, int index,
758			       __be16 ah_udp_s_port);
759int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port,
760			   int index, int *gid_type);
761struct net_device *mlx5_ib_get_netdev(struct ib_device *ib_dev, u8 port);
762int modify_gid_roce(struct ib_device *ib_dev, u8 port, unsigned int index,
763		    const union ib_gid *gid, struct net_device *ndev);
764int query_gid_roce(struct ib_device *ib_dev, u8 port, int index,
765		   union ib_gid *gid);
766int mlx5_process_mad_mad_ifc(struct ib_device *ibdev, int mad_flags,
767			     u8 port_num, struct ib_wc *in_wc,
768			     struct ib_grh *in_grh, struct ib_mad *in_mad,
769			     struct ib_mad *out_mad);
770
771static inline void init_query_mad(struct ib_smp *mad)
772{
773	mad->base_version  = 1;
774	mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
775	mad->class_version = 1;
776	mad->method	   = IB_MGMT_METHOD_GET;
777}
778
779static inline u8 convert_access(int acc)
780{
781	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
782	       (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
783	       (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
784	       (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
785	       MLX5_PERM_LOCAL_READ;
786}
787
788#define MLX5_MAX_UMR_SHIFT 16
789#define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
790
791#endif /* MLX5_IB_H */
792