mlx4.h revision 331769 
1/*
2 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses.  You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 *     Redistribution and use in source and binary forms, with or
12 *     without modification, are permitted provided that the following
13 *     conditions are met:
14 *
15 *      - Redistributions of source code must retain the above
16 *        copyright notice, this list of conditions and the following
17 *        disclaimer.
18 *
19 *      - Redistributions in binary form must reproduce the above
20 *        copyright notice, this list of conditions and the following
21 *        disclaimer in the documentation and/or other materials
22 *        provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#ifndef MLX4_H
35#define MLX4_H
36
37#include <infiniband/endian.h>
38#include <stddef.h>
39
40#include <infiniband/driver.h>
41#include <infiniband/udma_barrier.h>
42#include <infiniband/verbs.h>
43
44#define MLX4_PORTS_NUM 2
45
46#define PFX		"mlx4: "
47
48enum {
49	MLX4_STAT_RATE_OFFSET		= 5
50};
51
52enum {
53	MLX4_QP_TABLE_BITS		= 8,
54	MLX4_QP_TABLE_SIZE		= 1 << MLX4_QP_TABLE_BITS,
55	MLX4_QP_TABLE_MASK		= MLX4_QP_TABLE_SIZE - 1
56};
57
58#define MLX4_REMOTE_SRQN_FLAGS(wr) htobe32(wr->qp_type.xrc.remote_srqn << 8)
59
60enum {
61	MLX4_XSRQ_TABLE_BITS = 8,
62	MLX4_XSRQ_TABLE_SIZE = 1 << MLX4_XSRQ_TABLE_BITS,
63	MLX4_XSRQ_TABLE_MASK = MLX4_XSRQ_TABLE_SIZE - 1
64};
65
66struct mlx4_xsrq_table {
67	struct {
68		struct mlx4_srq **table;
69		int		  refcnt;
70	} xsrq_table[MLX4_XSRQ_TABLE_SIZE];
71
72	pthread_mutex_t		  mutex;
73	int			  num_xsrq;
74	int			  shift;
75	int			  mask;
76};
77
78enum {
79	MLX4_XRC_QPN_BIT     = (1 << 23)
80};
81
82enum mlx4_db_type {
83	MLX4_DB_TYPE_CQ,
84	MLX4_DB_TYPE_RQ,
85	MLX4_NUM_DB_TYPE
86};
87
88enum {
89	MLX4_OPCODE_NOP			= 0x00,
90	MLX4_OPCODE_SEND_INVAL		= 0x01,
91	MLX4_OPCODE_RDMA_WRITE		= 0x08,
92	MLX4_OPCODE_RDMA_WRITE_IMM	= 0x09,
93	MLX4_OPCODE_SEND		= 0x0a,
94	MLX4_OPCODE_SEND_IMM		= 0x0b,
95	MLX4_OPCODE_LSO			= 0x0e,
96	MLX4_OPCODE_RDMA_READ		= 0x10,
97	MLX4_OPCODE_ATOMIC_CS		= 0x11,
98	MLX4_OPCODE_ATOMIC_FA		= 0x12,
99	MLX4_OPCODE_MASKED_ATOMIC_CS	= 0x14,
100	MLX4_OPCODE_MASKED_ATOMIC_FA	= 0x15,
101	MLX4_OPCODE_BIND_MW		= 0x18,
102	MLX4_OPCODE_FMR			= 0x19,
103	MLX4_OPCODE_LOCAL_INVAL		= 0x1b,
104	MLX4_OPCODE_CONFIG_CMD		= 0x1f,
105
106	MLX4_RECV_OPCODE_RDMA_WRITE_IMM	= 0x00,
107	MLX4_RECV_OPCODE_SEND		= 0x01,
108	MLX4_RECV_OPCODE_SEND_IMM	= 0x02,
109	MLX4_RECV_OPCODE_SEND_INVAL	= 0x03,
110
111	MLX4_CQE_OPCODE_ERROR		= 0x1e,
112	MLX4_CQE_OPCODE_RESIZE		= 0x16,
113};
114
115struct mlx4_device {
116	struct verbs_device		verbs_dev;
117	int				page_size;
118	int				abi_version;
119};
120
121struct mlx4_db_page;
122
123struct mlx4_context {
124	struct ibv_context		ibv_ctx;
125
126	void			       *uar;
127	pthread_spinlock_t		uar_lock;
128
129	void			       *bf_page;
130	int				bf_buf_size;
131	int				bf_offset;
132	pthread_spinlock_t		bf_lock;
133
134	struct {
135		struct mlx4_qp	      **table;
136		int			refcnt;
137	}				qp_table[MLX4_QP_TABLE_SIZE];
138	pthread_mutex_t			qp_table_mutex;
139	int				num_qps;
140	int				qp_table_shift;
141	int				qp_table_mask;
142	int				max_qp_wr;
143	int				max_sge;
144
145	struct mlx4_db_page	       *db_list[MLX4_NUM_DB_TYPE];
146	pthread_mutex_t			db_list_mutex;
147	int				cqe_size;
148	struct mlx4_xsrq_table		xsrq_table;
149	struct {
150		uint8_t                 valid;
151		uint8_t                 link_layer;
152		enum ibv_port_cap_flags caps;
153	} port_query_cache[MLX4_PORTS_NUM];
154	struct {
155		uint64_t                offset;
156		uint8_t                 offset_valid;
157	} core_clock;
158	void			       *hca_core_clock;
159};
160
161struct mlx4_buf {
162	void			       *buf;
163	size_t				length;
164};
165
166struct mlx4_pd {
167	struct ibv_pd			ibv_pd;
168	uint32_t			pdn;
169};
170
171enum {
172	MLX4_CQ_FLAGS_RX_CSUM_VALID = 1 << 0,
173	MLX4_CQ_FLAGS_EXTENDED = 1 << 1,
174	MLX4_CQ_FLAGS_SINGLE_THREADED = 1 << 2,
175};
176
177struct mlx4_cq {
178	struct ibv_cq_ex		ibv_cq;
179	struct mlx4_buf			buf;
180	struct mlx4_buf			resize_buf;
181	pthread_spinlock_t		lock;
182	uint32_t			cqn;
183	uint32_t			cons_index;
184	uint32_t		       *set_ci_db;
185	uint32_t		       *arm_db;
186	int				arm_sn;
187	int				cqe_size;
188	struct mlx4_qp			*cur_qp;
189	struct mlx4_cqe			*cqe;
190	uint32_t			flags;
191};
192
193struct mlx4_srq {
194	struct verbs_srq		verbs_srq;
195	struct mlx4_buf			buf;
196	pthread_spinlock_t		lock;
197	uint64_t		       *wrid;
198	uint32_t			srqn;
199	int				max;
200	int				max_gs;
201	int				wqe_shift;
202	int				head;
203	int				tail;
204	uint32_t		       *db;
205	uint16_t			counter;
206	uint8_t				ext_srq;
207};
208
209struct mlx4_wq {
210	uint64_t		       *wrid;
211	pthread_spinlock_t		lock;
212	int				wqe_cnt;
213	int				max_post;
214	unsigned			head;
215	unsigned			tail;
216	int				max_gs;
217	int				wqe_shift;
218	int				offset;
219};
220
221struct mlx4_qp {
222	struct verbs_qp			verbs_qp;
223	struct mlx4_buf			buf;
224	int				max_inline_data;
225	int				buf_size;
226
227	uint32_t			doorbell_qpn;
228	uint32_t			sq_signal_bits;
229	int				sq_spare_wqes;
230	struct mlx4_wq			sq;
231
232	uint32_t		       *db;
233	struct mlx4_wq			rq;
234
235	uint8_t				link_layer;
236	uint32_t			qp_cap_cache;
237};
238
239struct mlx4_av {
240	uint32_t			port_pd;
241	uint8_t				reserved1;
242	uint8_t				g_slid;
243	uint16_t			dlid;
244	uint8_t				reserved2;
245	uint8_t				gid_index;
246	uint8_t				stat_rate;
247	uint8_t				hop_limit;
248	uint32_t			sl_tclass_flowlabel;
249	uint8_t				dgid[16];
250};
251
252struct mlx4_ah {
253	struct ibv_ah			ibv_ah;
254	struct mlx4_av			av;
255	uint16_t			vlan;
256	uint8_t				mac[6];
257};
258
259enum {
260	MLX4_CSUM_SUPPORT_UD_OVER_IB	= (1 <<  0),
261	MLX4_CSUM_SUPPORT_RAW_OVER_ETH	= (1 <<  1),
262	/* Only report rx checksum when the validation is valid */
263	MLX4_RX_CSUM_VALID		= (1 <<  16),
264};
265
266enum mlx4_cqe_status {
267	MLX4_CQE_STATUS_TCP_UDP_CSUM_OK	= (1 <<  2),
268	MLX4_CQE_STATUS_IPV4_PKT	= (1 << 22),
269	MLX4_CQE_STATUS_IP_HDR_CSUM_OK	= (1 << 28),
270	MLX4_CQE_STATUS_IPV4_CSUM_OK	= MLX4_CQE_STATUS_IPV4_PKT |
271					MLX4_CQE_STATUS_IP_HDR_CSUM_OK |
272					MLX4_CQE_STATUS_TCP_UDP_CSUM_OK
273};
274
275struct mlx4_cqe {
276	uint32_t	vlan_my_qpn;
277	uint32_t	immed_rss_invalid;
278	uint32_t	g_mlpath_rqpn;
279	union {
280		struct {
281			uint16_t	sl_vid;
282			uint16_t	rlid;
283		};
284		uint32_t ts_47_16;
285	};
286	uint32_t	status;
287	uint32_t	byte_cnt;
288	uint16_t	wqe_index;
289	uint16_t	checksum;
290	uint8_t		reserved3;
291	uint8_t		ts_15_8;
292	uint8_t		ts_7_0;
293	uint8_t		owner_sr_opcode;
294};
295
296static inline unsigned long align(unsigned long val, unsigned long align)
297{
298	return (val + align - 1) & ~(align - 1);
299}
300int align_queue_size(int req);
301
302#define to_mxxx(xxx, type)						\
303	((struct mlx4_##type *)					\
304	 ((void *) ib##xxx - offsetof(struct mlx4_##type, ibv_##xxx)))
305
306static inline struct mlx4_device *to_mdev(struct ibv_device *ibdev)
307{
308	/* ibv_device is first field of verbs_device
309	 * see try_driver() in libibverbs.
310	 */
311	return container_of(ibdev, struct mlx4_device, verbs_dev);
312}
313
314static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx)
315{
316	return to_mxxx(ctx, context);
317}
318
319static inline struct mlx4_pd *to_mpd(struct ibv_pd *ibpd)
320{
321	return to_mxxx(pd, pd);
322}
323
324static inline struct mlx4_cq *to_mcq(struct ibv_cq *ibcq)
325{
326	return to_mxxx(cq, cq);
327}
328
329static inline struct mlx4_srq *to_msrq(struct ibv_srq *ibsrq)
330{
331	return container_of(container_of(ibsrq, struct verbs_srq, srq),
332			    struct mlx4_srq, verbs_srq);
333}
334
335static inline struct mlx4_qp *to_mqp(struct ibv_qp *ibqp)
336{
337	return container_of(container_of(ibqp, struct verbs_qp, qp),
338			    struct mlx4_qp, verbs_qp);
339}
340
341static inline struct mlx4_ah *to_mah(struct ibv_ah *ibah)
342{
343	return to_mxxx(ah, ah);
344}
345
346static inline void mlx4_update_cons_index(struct mlx4_cq *cq)
347{
348	*cq->set_ci_db = htobe32(cq->cons_index & 0xffffff);
349}
350
351int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size);
352void mlx4_free_buf(struct mlx4_buf *buf);
353
354uint32_t *mlx4_alloc_db(struct mlx4_context *context, enum mlx4_db_type type);
355void mlx4_free_db(struct mlx4_context *context, enum mlx4_db_type type, uint32_t *db);
356
357int mlx4_query_device(struct ibv_context *context,
358		       struct ibv_device_attr *attr);
359int mlx4_query_device_ex(struct ibv_context *context,
360			 const struct ibv_query_device_ex_input *input,
361			 struct ibv_device_attr_ex *attr,
362			 size_t attr_size);
363int mlx4_query_port(struct ibv_context *context, uint8_t port,
364		     struct ibv_port_attr *attr);
365int mlx4_query_rt_values(struct ibv_context *context,
366			 struct ibv_values_ex *values);
367struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context);
368int mlx4_free_pd(struct ibv_pd *pd);
369struct ibv_xrcd *mlx4_open_xrcd(struct ibv_context *context,
370				struct ibv_xrcd_init_attr *attr);
371int mlx4_close_xrcd(struct ibv_xrcd *xrcd);
372
373struct ibv_mr *mlx4_reg_mr(struct ibv_pd *pd, void *addr,
374			    size_t length, int access);
375int mlx4_rereg_mr(struct ibv_mr *mr, int flags, struct ibv_pd *pd,
376		  void *addr, size_t length, int access);
377int mlx4_dereg_mr(struct ibv_mr *mr);
378
379struct ibv_mw *mlx4_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type);
380int mlx4_dealloc_mw(struct ibv_mw *mw);
381int mlx4_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
382		 struct ibv_mw_bind *mw_bind);
383
384struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe,
385			       struct ibv_comp_channel *channel,
386			       int comp_vector);
387struct ibv_cq_ex *mlx4_create_cq_ex(struct ibv_context *context,
388				    struct ibv_cq_init_attr_ex *cq_attr);
389void mlx4_cq_fill_pfns(struct mlx4_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr);
390int mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent,
391		      int entry_size);
392int mlx4_resize_cq(struct ibv_cq *cq, int cqe);
393int mlx4_destroy_cq(struct ibv_cq *cq);
394int mlx4_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
395int mlx4_arm_cq(struct ibv_cq *cq, int solicited);
396void mlx4_cq_event(struct ibv_cq *cq);
397void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
398void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
399int mlx4_get_outstanding_cqes(struct mlx4_cq *cq);
400void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int new_cqe);
401
402struct ibv_srq *mlx4_create_srq(struct ibv_pd *pd,
403				 struct ibv_srq_init_attr *attr);
404struct ibv_srq *mlx4_create_srq_ex(struct ibv_context *context,
405				   struct ibv_srq_init_attr_ex *attr_ex);
406struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context,
407				    struct ibv_srq_init_attr_ex *attr_ex);
408int mlx4_modify_srq(struct ibv_srq *srq,
409		     struct ibv_srq_attr *attr,
410		     int mask);
411int mlx4_query_srq(struct ibv_srq *srq,
412			   struct ibv_srq_attr *attr);
413int mlx4_destroy_srq(struct ibv_srq *srq);
414int mlx4_destroy_xrc_srq(struct ibv_srq *srq);
415int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
416			struct mlx4_srq *srq);
417void mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size);
418struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn);
419int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn,
420		    struct mlx4_srq *srq);
421void mlx4_clear_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn);
422void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);
423int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
424		       struct ibv_recv_wr *wr,
425		       struct ibv_recv_wr **bad_wr);
426
427struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
428struct ibv_qp *mlx4_create_qp_ex(struct ibv_context *context,
429				 struct ibv_qp_init_attr_ex *attr);
430struct ibv_qp *mlx4_open_qp(struct ibv_context *context, struct ibv_qp_open_attr *attr);
431int mlx4_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
432		   int attr_mask,
433		   struct ibv_qp_init_attr *init_attr);
434int mlx4_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
435		    int attr_mask);
436int mlx4_destroy_qp(struct ibv_qp *qp);
437void mlx4_init_qp_indices(struct mlx4_qp *qp);
438void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp);
439int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
440			  struct ibv_send_wr **bad_wr);
441int mlx4_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
442			  struct ibv_recv_wr **bad_wr);
443void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
444			   struct mlx4_qp *qp);
445int mlx4_alloc_qp_buf(struct ibv_context *context, struct ibv_qp_cap *cap,
446		       enum ibv_qp_type type, struct mlx4_qp *qp);
447void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
448		       enum ibv_qp_type type);
449struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn);
450int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp);
451void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn);
452struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
453int mlx4_destroy_ah(struct ibv_ah *ah);
454int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr,
455		   struct mlx4_ah *ah);
456void mlx4_free_av(struct mlx4_ah *ah);
457
458#endif /* MLX4_H */
459