iw_cxgbe.h revision 331769 
1/*
2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *	  copyright notice, this list of conditions and the following
16 *	  disclaimer.
17 *      - Redistributions in binary form must reproduce the above
18 *	  copyright notice, this list of conditions and the following
19 *	  disclaimer in the documentation and/or other materials
20 *	  provided with the distribution.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29 * SOFTWARE.
30 *
31 * $FreeBSD: stable/11/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h 331769 2018-03-30 18:06:29Z hselasky $
32 */
33#ifndef __IW_CXGB4_H__
34#define __IW_CXGB4_H__
35
36#include <linux/list.h>
37#include <linux/spinlock.h>
38#include <linux/idr.h>
39#include <linux/completion.h>
40#include <linux/netdevice.h>
41#include <linux/sched.h>
42#include <linux/pci.h>
43#include <linux/dma-mapping.h>
44#include <linux/wait.h>
45#include <linux/kref.h>
46#include <linux/timer.h>
47#include <linux/io.h>
48#include <sys/vmem.h>
49
50#include <asm/byteorder.h>
51
52#include <netinet/in.h>
53#include <netinet/toecore.h>
54
55#include <rdma/ib_verbs.h>
56#include <rdma/iw_cm.h>
57
58#undef prefetch
59
60#include "common/common.h"
61#include "common/t4_msg.h"
62#include "common/t4_regs.h"
63#include "common/t4_tcb.h"
64#include "t4_l2t.h"
65
66#define DRV_NAME "iw_cxgbe"
67#define MOD DRV_NAME ":"
68#define KTR_IW_CXGBE	KTR_SPARE3
69
70extern int c4iw_debug;
71#define PDBG(fmt, args...) \
72do { \
73	if (c4iw_debug) \
74		printf(MOD fmt, ## args); \
75} while (0)
76
77#include "t4.h"
78
79static inline void *cplhdr(struct mbuf *m)
80{
81	return mtod(m, void*);
82}
83
84#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.pbl.start)
85#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.rq.start)
86
87#define C4IW_ID_TABLE_F_RANDOM 1       /* Pseudo-randomize the id's returned */
88#define C4IW_ID_TABLE_F_EMPTY  2       /* Table is initially empty */
89
90struct c4iw_id_table {
91	u32 flags;
92	u32 start;              /* logical minimal id */
93	u32 last;               /* hint for find */
94	u32 max;
95	spinlock_t lock;
96	unsigned long *table;
97};
98
99struct c4iw_resource {
100	struct c4iw_id_table tpt_table;
101	struct c4iw_id_table qid_table;
102	struct c4iw_id_table pdid_table;
103};
104
105struct c4iw_qid_list {
106	struct list_head entry;
107	u32 qid;
108};
109
110struct c4iw_dev_ucontext {
111	struct list_head qpids;
112	struct list_head cqids;
113	struct mutex lock;
114};
115
116enum c4iw_rdev_flags {
117	T4_FATAL_ERROR = (1<<0),
118	T4_STATUS_PAGE_DISABLED = (1<<1),
119};
120
121struct c4iw_stat {
122	u64 total;
123	u64 cur;
124	u64 max;
125	u64 fail;
126};
127
128struct c4iw_stats {
129	struct mutex lock;
130	struct c4iw_stat qid;
131	struct c4iw_stat pd;
132	struct c4iw_stat stag;
133	struct c4iw_stat pbl;
134	struct c4iw_stat rqt;
135};
136
137struct c4iw_hw_queue {
138	int t4_eq_status_entries;
139	int t4_max_eq_size;
140	int t4_max_iq_size;
141	int t4_max_rq_size;
142	int t4_max_sq_size;
143	int t4_max_qp_depth;
144	int t4_max_cq_depth;
145	int t4_stat_len;
146};
147
148struct c4iw_rdev {
149	struct adapter *adap;
150	struct c4iw_resource resource;
151	unsigned long qpshift;
152	u32 qpmask;
153	unsigned long cqshift;
154	u32 cqmask;
155	struct c4iw_dev_ucontext uctx;
156	vmem_t          *rqt_arena;
157	vmem_t          *pbl_arena;
158	u32 flags;
159	struct c4iw_stats stats;
160	struct c4iw_hw_queue hw_queue;
161	struct t4_dev_status_page *status_page;
162	unsigned long bar2_pa;
163	void __iomem *bar2_kva;
164	unsigned int bar2_len;
165	struct workqueue_struct *free_workq;
166};
167
168static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
169{
170	return rdev->flags & T4_FATAL_ERROR;
171}
172
173static inline int c4iw_num_stags(struct c4iw_rdev *rdev)
174{
175	return (int)(rdev->adap->vres.stag.size >> 5);
176}
177
178#define C4IW_WR_TO (60*HZ)
179
180struct c4iw_wr_wait {
181	int ret;
182	struct completion completion;
183};
184
185static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
186{
187	wr_waitp->ret = 0;
188	init_completion(&wr_waitp->completion);
189}
190
191static inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret)
192{
193	wr_waitp->ret = ret;
194	complete(&wr_waitp->completion);
195}
196
197static inline int
198c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp,
199		u32 hwtid, u32 qpid, struct socket *so, const char *func)
200{
201	struct adapter *sc = rdev->adap;
202	unsigned to = C4IW_WR_TO;
203	int ret;
204	int timedout = 0;
205	struct timeval t1, t2;
206
207	if (c4iw_fatal_error(rdev)) {
208		wr_waitp->ret = -EIO;
209		goto out;
210	}
211
212	getmicrotime(&t1);
213	do {
214		/* If waiting for reply in rdma_init()/rdma_fini() threads, then
215		 * check if there are any connection errors.
216		 */
217		if (so && so->so_error) {
218			wr_waitp->ret = -ECONNRESET;
219			CTR5(KTR_IW_CXGBE, "%s - Connection ERROR %u for sock %p"
220			    "tid %u qpid %u", func,
221			    so->so_error, so, hwtid, qpid);
222			break;
223		}
224
225		ret = wait_for_completion_timeout(&wr_waitp->completion, to);
226		if (!ret) {
227			getmicrotime(&t2);
228			timevalsub(&t2, &t1);
229			printf("%s - Device %s not responding after %ld.%06ld "
230			    "seconds - tid %u qpid %u\n", func,
231			    device_get_nameunit(sc->dev), t2.tv_sec, t2.tv_usec,
232			    hwtid, qpid);
233			if (c4iw_fatal_error(rdev)) {
234				wr_waitp->ret = -EIO;
235				break;
236			}
237			to = to << 2;
238			timedout = 1;
239		}
240	} while (!ret);
241
242out:
243	if (timedout) {
244		getmicrotime(&t2);
245		timevalsub(&t2, &t1);
246		printf("%s - Device %s reply after %ld.%06ld seconds - "
247		    "tid %u qpid %u\n", func, device_get_nameunit(sc->dev),
248		    t2.tv_sec, t2.tv_usec, hwtid, qpid);
249	}
250	if (wr_waitp->ret)
251		CTR4(KTR_IW_CXGBE, "%p: FW reply %d tid %u qpid %u", sc,
252		    wr_waitp->ret, hwtid, qpid);
253	return (wr_waitp->ret);
254}
255
256struct c4iw_dev {
257	struct ib_device ibdev;
258	struct c4iw_rdev rdev;
259	u32 device_cap_flags;
260	struct idr cqidr;
261	struct idr qpidr;
262	struct idr mmidr;
263	spinlock_t lock;
264	struct dentry *debugfs_root;
265	u32 avail_ird;
266};
267
268static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
269{
270	return container_of(ibdev, struct c4iw_dev, ibdev);
271}
272
273static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
274{
275	return container_of(rdev, struct c4iw_dev, rdev);
276}
277
278static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
279{
280	return idr_find(&rhp->cqidr, cqid);
281}
282
283static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
284{
285	return idr_find(&rhp->qpidr, qpid);
286}
287
288static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid)
289{
290	return idr_find(&rhp->mmidr, mmid);
291}
292
293static inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr,
294				 void *handle, u32 id, int lock)
295{
296	int ret;
297	int newid;
298
299	do {
300		if (!idr_pre_get(idr, lock ? GFP_KERNEL : GFP_ATOMIC))
301			return -ENOMEM;
302		if (lock)
303			spin_lock_irq(&rhp->lock);
304		ret = idr_get_new_above(idr, handle, id, &newid);
305		BUG_ON(!ret && newid != id);
306		if (lock)
307			spin_unlock_irq(&rhp->lock);
308	} while (ret == -EAGAIN);
309
310	return ret;
311}
312
313static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
314				void *handle, u32 id)
315{
316	return _insert_handle(rhp, idr, handle, id, 1);
317}
318
319static inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr,
320				       void *handle, u32 id)
321{
322	return _insert_handle(rhp, idr, handle, id, 0);
323}
324
325static inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr,
326				   u32 id, int lock)
327{
328	if (lock)
329		spin_lock_irq(&rhp->lock);
330	idr_remove(idr, id);
331	if (lock)
332		spin_unlock_irq(&rhp->lock);
333}
334
335static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
336{
337	_remove_handle(rhp, idr, id, 1);
338}
339
340static inline void remove_handle_nolock(struct c4iw_dev *rhp,
341					 struct idr *idr, u32 id)
342{
343	_remove_handle(rhp, idr, id, 0);
344}
345
346extern int c4iw_max_read_depth;
347
348static inline int cur_max_read_depth(struct c4iw_dev *dev)
349{
350	return min(dev->rdev.adap->params.max_ordird_qp, c4iw_max_read_depth);
351}
352
353struct c4iw_pd {
354	struct ib_pd ibpd;
355	u32 pdid;
356	struct c4iw_dev *rhp;
357};
358
359static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)
360{
361	return container_of(ibpd, struct c4iw_pd, ibpd);
362}
363
364struct tpt_attributes {
365	u64 len;
366	u64 va_fbo;
367	enum fw_ri_mem_perms perms;
368	u32 stag;
369	u32 pdid;
370	u32 qpid;
371	u32 pbl_addr;
372	u32 pbl_size;
373	u32 state:1;
374	u32 type:2;
375	u32 rsvd:1;
376	u32 remote_invaliate_disable:1;
377	u32 zbva:1;
378	u32 mw_bind_enable:1;
379	u32 page_size:5;
380};
381
382struct c4iw_mr {
383	struct ib_mr ibmr;
384	struct ib_umem *umem;
385	struct c4iw_dev *rhp;
386	u64 kva;
387	struct tpt_attributes attr;
388	u64 *mpl;
389	dma_addr_t mpl_addr;
390	u32 max_mpl_len;
391	u32 mpl_len;
392};
393
394static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
395{
396	return container_of(ibmr, struct c4iw_mr, ibmr);
397}
398
399struct c4iw_mw {
400	struct ib_mw ibmw;
401	struct c4iw_dev *rhp;
402	u64 kva;
403	struct tpt_attributes attr;
404};
405
406static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
407{
408	return container_of(ibmw, struct c4iw_mw, ibmw);
409}
410
411struct c4iw_cq {
412	struct ib_cq ibcq;
413	struct c4iw_dev *rhp;
414	struct t4_cq cq;
415	spinlock_t lock;
416	spinlock_t comp_handler_lock;
417	atomic_t refcnt;
418	wait_queue_head_t wait;
419};
420
421static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)
422{
423	return container_of(ibcq, struct c4iw_cq, ibcq);
424}
425
426struct c4iw_mpa_attributes {
427	u8 initiator;
428	u8 recv_marker_enabled;
429	u8 xmit_marker_enabled;
430	u8 crc_enabled;
431	u8 enhanced_rdma_conn;
432	u8 version;
433	u8 p2p_type;
434};
435
436struct c4iw_qp_attributes {
437	u32 scq;
438	u32 rcq;
439	u32 sq_num_entries;
440	u32 rq_num_entries;
441	u32 sq_max_sges;
442	u32 sq_max_sges_rdma_write;
443	u32 rq_max_sges;
444	u32 state;
445	u8 enable_rdma_read;
446	u8 enable_rdma_write;
447	u8 enable_bind;
448	u8 enable_mmid0_fastreg;
449	u32 max_ord;
450	u32 max_ird;
451	u32 pd;
452	u32 next_state;
453	char terminate_buffer[52];
454	u32 terminate_msg_len;
455	u8 is_terminate_local;
456	struct c4iw_mpa_attributes mpa_attr;
457	struct c4iw_ep *llp_stream_handle;
458	u8 layer_etype;
459	u8 ecode;
460	u16 sq_db_inc;
461	u16 rq_db_inc;
462	u8 send_term;
463};
464
465struct c4iw_qp {
466	struct ib_qp ibqp;
467	struct c4iw_dev *rhp;
468	struct c4iw_ep *ep;
469	struct c4iw_qp_attributes attr;
470	struct t4_wq wq;
471	spinlock_t lock;
472	struct mutex mutex;
473	struct kref kref;
474	wait_queue_head_t wait;
475	struct timer_list timer;
476	int sq_sig_all;
477	struct work_struct free_work;
478	struct c4iw_ucontext *ucontext;
479};
480
481static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
482{
483	return container_of(ibqp, struct c4iw_qp, ibqp);
484}
485
486struct c4iw_ucontext {
487	struct ib_ucontext ibucontext;
488	struct c4iw_dev_ucontext uctx;
489	u32 key;
490	spinlock_t mmap_lock;
491	struct list_head mmaps;
492	struct kref kref;
493};
494
495static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
496{
497	return container_of(c, struct c4iw_ucontext, ibucontext);
498}
499
500void _c4iw_free_ucontext(struct kref *kref);
501
502static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext)
503{
504	kref_put(&ucontext->kref, _c4iw_free_ucontext);
505}
506static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext)
507{
508	kref_get(&ucontext->kref);
509}
510
511struct c4iw_mm_entry {
512	struct list_head entry;
513	u64 addr;
514	u32 key;
515	unsigned len;
516};
517
518static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,
519						u32 key, unsigned len)
520{
521	struct list_head *pos, *nxt;
522	struct c4iw_mm_entry *mm;
523
524	spin_lock(&ucontext->mmap_lock);
525	list_for_each_safe(pos, nxt, &ucontext->mmaps) {
526
527		mm = list_entry(pos, struct c4iw_mm_entry, entry);
528		if (mm->key == key && mm->len == len) {
529			list_del_init(&mm->entry);
530			spin_unlock(&ucontext->mmap_lock);
531			CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d",
532			     __func__, key, (unsigned long long) mm->addr,
533			     mm->len);
534			return mm;
535		}
536	}
537	spin_unlock(&ucontext->mmap_lock);
538	return NULL;
539}
540
541static inline void insert_mmap(struct c4iw_ucontext *ucontext,
542			       struct c4iw_mm_entry *mm)
543{
544	spin_lock(&ucontext->mmap_lock);
545	CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", __func__, mm->key,
546	    (unsigned long long) mm->addr, mm->len);
547	list_add_tail(&mm->entry, &ucontext->mmaps);
548	spin_unlock(&ucontext->mmap_lock);
549}
550
551enum c4iw_qp_attr_mask {
552	C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
553	C4IW_QP_ATTR_SQ_DB = 1<<1,
554	C4IW_QP_ATTR_RQ_DB = 1<<2,
555	C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
556	C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
557	C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
558	C4IW_QP_ATTR_MAX_ORD = 1 << 11,
559	C4IW_QP_ATTR_MAX_IRD = 1 << 12,
560	C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,
561	C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,
562	C4IW_QP_ATTR_MPA_ATTR = 1 << 24,
563	C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,
564	C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |
565				     C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
566				     C4IW_QP_ATTR_MAX_ORD |
567				     C4IW_QP_ATTR_MAX_IRD |
568				     C4IW_QP_ATTR_LLP_STREAM_HANDLE |
569				     C4IW_QP_ATTR_STREAM_MSG_BUFFER |
570				     C4IW_QP_ATTR_MPA_ATTR |
571				     C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)
572};
573
574int c4iw_modify_qp(struct c4iw_dev *rhp,
575				struct c4iw_qp *qhp,
576				enum c4iw_qp_attr_mask mask,
577				struct c4iw_qp_attributes *attrs,
578				int internal);
579
580enum c4iw_qp_state {
581	C4IW_QP_STATE_IDLE,
582	C4IW_QP_STATE_RTS,
583	C4IW_QP_STATE_ERROR,
584	C4IW_QP_STATE_TERMINATE,
585	C4IW_QP_STATE_CLOSING,
586	C4IW_QP_STATE_TOT
587};
588
589/*
590 * IW_CXGBE event bits.
591 * These bits are used for handling all events for a particular 'ep' serially.
592 */
593#define	C4IW_EVENT_SOCKET	0x0001
594#define	C4IW_EVENT_TIMEOUT	0x0002
595#define	C4IW_EVENT_TERM		0x0004
596
597static inline int c4iw_convert_state(enum ib_qp_state ib_state)
598{
599	switch (ib_state) {
600	case IB_QPS_RESET:
601	case IB_QPS_INIT:
602		return C4IW_QP_STATE_IDLE;
603	case IB_QPS_RTS:
604		return C4IW_QP_STATE_RTS;
605	case IB_QPS_SQD:
606		return C4IW_QP_STATE_CLOSING;
607	case IB_QPS_SQE:
608		return C4IW_QP_STATE_TERMINATE;
609	case IB_QPS_ERR:
610		return C4IW_QP_STATE_ERROR;
611	default:
612		return -1;
613	}
614}
615
616static inline int to_ib_qp_state(int c4iw_qp_state)
617{
618	switch (c4iw_qp_state) {
619	case C4IW_QP_STATE_IDLE:
620		return IB_QPS_INIT;
621	case C4IW_QP_STATE_RTS:
622		return IB_QPS_RTS;
623	case C4IW_QP_STATE_CLOSING:
624		return IB_QPS_SQD;
625	case C4IW_QP_STATE_TERMINATE:
626		return IB_QPS_SQE;
627	case C4IW_QP_STATE_ERROR:
628		return IB_QPS_ERR;
629	}
630	return IB_QPS_ERR;
631}
632
633#define C4IW_DRAIN_OPCODE FW_RI_SGE_EC_CR_RETURN
634
635static inline u32 c4iw_ib_to_tpt_access(int a)
636{
637	return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
638	       (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |
639	       (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |
640	       FW_RI_MEM_ACCESS_LOCAL_READ;
641}
642
643static inline u32 c4iw_ib_to_tpt_bind_access(int acc)
644{
645	return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
646	       (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0);
647}
648
649enum c4iw_mmid_state {
650	C4IW_STAG_STATE_VALID,
651	C4IW_STAG_STATE_INVALID
652};
653
654#define C4IW_NODE_DESC "iw_cxgbe Chelsio Communications"
655
656#define MPA_KEY_REQ "MPA ID Req Frame"
657#define MPA_KEY_REP "MPA ID Rep Frame"
658
659#define MPA_MAX_PRIVATE_DATA	256
660#define MPA_ENHANCED_RDMA_CONN	0x10
661#define MPA_REJECT		0x20
662#define MPA_CRC			0x40
663#define MPA_MARKERS		0x80
664#define MPA_FLAGS_MASK		0xE0
665
666#define MPA_V2_PEER2PEER_MODEL          0x8000
667#define MPA_V2_ZERO_LEN_FPDU_RTR        0x4000
668#define MPA_V2_RDMA_WRITE_RTR           0x8000
669#define MPA_V2_RDMA_READ_RTR            0x4000
670#define MPA_V2_IRD_ORD_MASK             0x3FFF
671
672#define c4iw_put_ep(ep) { \
673	CTR4(KTR_IW_CXGBE, "put_ep (%s:%u) ep %p, refcnt %d", \
674	     __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \
675	WARN_ON(atomic_read(&(ep)->kref.refcount) < 1); \
676        kref_put(&((ep)->kref), _c4iw_free_ep); \
677}
678
679#define c4iw_get_ep(ep) { \
680	CTR4(KTR_IW_CXGBE, "get_ep (%s:%u) ep %p, refcnt %d", \
681	      __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \
682        kref_get(&((ep)->kref));  \
683}
684
685void _c4iw_free_ep(struct kref *kref);
686
687struct mpa_message {
688	u8 key[16];
689	u8 flags;
690	u8 revision;
691	__be16 private_data_size;
692	u8 private_data[0];
693};
694
695struct mpa_v2_conn_params {
696	__be16 ird;
697	__be16 ord;
698};
699
700struct terminate_message {
701	u8 layer_etype;
702	u8 ecode;
703	__be16 hdrct_rsvd;
704	u8 len_hdrs[0];
705};
706
707#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)
708
709enum c4iw_layers_types {
710	LAYER_RDMAP		= 0x00,
711	LAYER_DDP		= 0x10,
712	LAYER_MPA		= 0x20,
713	RDMAP_LOCAL_CATA	= 0x00,
714	RDMAP_REMOTE_PROT	= 0x01,
715	RDMAP_REMOTE_OP		= 0x02,
716	DDP_LOCAL_CATA		= 0x00,
717	DDP_TAGGED_ERR		= 0x01,
718	DDP_UNTAGGED_ERR	= 0x02,
719	DDP_LLP			= 0x03
720};
721
722enum c4iw_rdma_ecodes {
723	RDMAP_INV_STAG		= 0x00,
724	RDMAP_BASE_BOUNDS	= 0x01,
725	RDMAP_ACC_VIOL		= 0x02,
726	RDMAP_STAG_NOT_ASSOC	= 0x03,
727	RDMAP_TO_WRAP		= 0x04,
728	RDMAP_INV_VERS		= 0x05,
729	RDMAP_INV_OPCODE	= 0x06,
730	RDMAP_STREAM_CATA	= 0x07,
731	RDMAP_GLOBAL_CATA	= 0x08,
732	RDMAP_CANT_INV_STAG	= 0x09,
733	RDMAP_UNSPECIFIED	= 0xff
734};
735
736enum c4iw_ddp_ecodes {
737	DDPT_INV_STAG		= 0x00,
738	DDPT_BASE_BOUNDS	= 0x01,
739	DDPT_STAG_NOT_ASSOC	= 0x02,
740	DDPT_TO_WRAP		= 0x03,
741	DDPT_INV_VERS		= 0x04,
742	DDPU_INV_QN		= 0x01,
743	DDPU_INV_MSN_NOBUF	= 0x02,
744	DDPU_INV_MSN_RANGE	= 0x03,
745	DDPU_INV_MO		= 0x04,
746	DDPU_MSG_TOOBIG		= 0x05,
747	DDPU_INV_VERS		= 0x06
748};
749
750enum c4iw_mpa_ecodes {
751	MPA_CRC_ERR		= 0x02,
752	MPA_MARKER_ERR		= 0x03,
753	MPA_LOCAL_CATA          = 0x05,
754	MPA_INSUFF_IRD          = 0x06,
755	MPA_NOMATCH_RTR         = 0x07,
756};
757
758enum c4iw_ep_state {
759	IDLE = 0,
760	LISTEN,
761	CONNECTING,
762	MPA_REQ_WAIT,
763	MPA_REQ_SENT,
764	MPA_REQ_RCVD,
765	MPA_REP_SENT,
766	FPDU_MODE,
767	ABORTING,
768	CLOSING,
769	MORIBUND,
770	DEAD,
771};
772
773enum c4iw_ep_flags {
774	PEER_ABORT_IN_PROGRESS	= 0,
775	ABORT_REQ_IN_PROGRESS	= 1,
776	RELEASE_RESOURCES	= 2,
777	CLOSE_SENT		= 3,
778	TIMEOUT                 = 4,
779	QP_REFERENCED		= 5,
780	STOP_MPA_TIMER		= 7,
781};
782
783enum c4iw_ep_history {
784        ACT_OPEN_REQ            = 0,
785        ACT_OFLD_CONN           = 1,
786        ACT_OPEN_RPL            = 2,
787        ACT_ESTAB               = 3,
788        PASS_ACCEPT_REQ         = 4,
789        PASS_ESTAB              = 5,
790        ABORT_UPCALL            = 6,
791        ESTAB_UPCALL            = 7,
792        CLOSE_UPCALL            = 8,
793        ULP_ACCEPT              = 9,
794        ULP_REJECT              = 10,
795        TIMEDOUT                = 11,
796        PEER_ABORT              = 12,
797        PEER_CLOSE              = 13,
798        CONNREQ_UPCALL          = 14,
799        ABORT_CONN              = 15,
800        DISCONN_UPCALL          = 16,
801        EP_DISC_CLOSE           = 17,
802        EP_DISC_ABORT           = 18,
803        CONN_RPL_UPCALL         = 19,
804        ACT_RETRY_NOMEM         = 20,
805        ACT_RETRY_INUSE         = 21,
806        CLOSE_CON_RPL           = 22,
807        EP_DISC_FAIL            = 24,
808        QP_REFED                = 25,
809        QP_DEREFED              = 26,
810        CM_ID_REFED             = 27,
811        CM_ID_DEREFED           = 28
812};
813
814struct c4iw_ep_common {
815	TAILQ_ENTRY(c4iw_ep_common) entry;	/* Work queue attachment */
816	struct iw_cm_id *cm_id;
817	struct c4iw_qp *qp;
818	struct c4iw_dev *dev;
819	enum c4iw_ep_state state;
820	struct kref kref;
821	struct mutex mutex;
822	struct sockaddr_storage local_addr;
823	struct sockaddr_storage remote_addr;
824	struct c4iw_wr_wait wr_wait;
825	unsigned long flags;
826	unsigned long history;
827        int rpl_err;
828        int rpl_done;
829        struct thread *thread;
830        struct socket *so;
831	int ep_events;
832};
833
834struct c4iw_listen_ep {
835	struct c4iw_ep_common com;
836	unsigned int stid;
837	int backlog;
838	struct list_head listen_ep_list;  /* list of all listener ep's bound
839					     to one port address */
840};
841
842struct c4iw_ep {
843	struct c4iw_ep_common com;
844	struct c4iw_listen_ep *parent_ep;
845	struct timer_list timer;
846	unsigned int atid;
847	u32 hwtid;
848	u32 snd_seq;
849	u32 rcv_seq;
850	struct l2t_entry *l2t;
851	struct dst_entry *dst;
852	struct c4iw_mpa_attributes mpa_attr;
853	u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];
854	unsigned int mpa_pkt_len;
855	u32 ird;
856	u32 ord;
857	u32 smac_idx;
858	u32 tx_chan;
859	u32 mtu;
860	u16 mss;
861	u16 emss;
862	u16 plen;
863	u16 rss_qid;
864	u16 txq_idx;
865	u16 ctrlq_idx;
866	u8 tos;
867	u8 retry_with_mpa_v1;
868	u8 tried_with_mpa_v1;
869};
870
871static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
872{
873	return cm_id->provider_data;
874}
875
876static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
877{
878	return cm_id->provider_data;
879}
880
881static inline int compute_wscale(int win)
882{
883	int wscale = 0;
884
885	while (wscale < 14 && (65535<<wscale) < win)
886		wscale++;
887	return wscale;
888}
889
890u32 c4iw_id_alloc(struct c4iw_id_table *alloc);
891void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);
892int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
893			u32 reserved, u32 flags);
894void c4iw_id_table_free(struct c4iw_id_table *alloc);
895
896typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct mbuf *m);
897
898int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
899		     struct l2t_entry *l2t);
900void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,
901		   struct c4iw_dev_ucontext *uctx);
902u32 c4iw_get_resource(struct c4iw_id_table *id_table);
903void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
904int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid);
905int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);
906int c4iw_pblpool_create(struct c4iw_rdev *rdev);
907int c4iw_rqtpool_create(struct c4iw_rdev *rdev);
908void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);
909void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);
910void c4iw_destroy_resource(struct c4iw_resource *rscp);
911int c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev);
912int c4iw_register_device(struct c4iw_dev *dev);
913void c4iw_unregister_device(struct c4iw_dev *dev);
914int __init c4iw_cm_init(void);
915void __exit c4iw_cm_term(void);
916void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
917			       struct c4iw_dev_ucontext *uctx);
918void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
919			    struct c4iw_dev_ucontext *uctx);
920int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
921int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
922		      struct ib_send_wr **bad_wr);
923int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
924		      struct ib_recv_wr **bad_wr);
925int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
926int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
927int c4iw_destroy_listen(struct iw_cm_id *cm_id);
928int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
929int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
930void c4iw_qp_add_ref(struct ib_qp *qp);
931void c4iw_qp_rem_ref(struct ib_qp *qp);
932struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
933			u32 max_num_sg);
934int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
935		int sg_nents, unsigned int *sg_offset);
936int c4iw_dealloc_mw(struct ib_mw *mw);
937struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
938		struct ib_udata *udata);
939struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64
940		virt, int acc, struct ib_udata *udata);
941struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
942int c4iw_dereg_mr(struct ib_mr *ib_mr);
943void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
944int c4iw_destroy_cq(struct ib_cq *ib_cq);
945struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
946				const struct ib_cq_init_attr *attr,
947				struct ib_ucontext *ib_context,
948				struct ib_udata *udata);
949int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
950int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
951int c4iw_destroy_qp(struct ib_qp *ib_qp);
952struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
953			     struct ib_qp_init_attr *attrs,
954			     struct ib_udata *udata);
955int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
956				 int attr_mask, struct ib_udata *udata);
957int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
958		     int attr_mask, struct ib_qp_init_attr *init_attr);
959struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
960u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
961void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
962u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
963void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
964int c4iw_ofld_send(struct c4iw_rdev *rdev, struct mbuf *m);
965void c4iw_flush_hw_cq(struct c4iw_cq *cq);
966void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
967void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
968int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
969int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
970int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
971int c4iw_flush_sq(struct c4iw_qp *qhp);
972int c4iw_ev_handler(struct sge_iq *, const struct rsp_ctrl *);
973u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
974int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
975u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
976void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
977		struct c4iw_dev_ucontext *uctx);
978u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
979void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
980		struct c4iw_dev_ucontext *uctx);
981void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
982void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
983		enum t4_bar2_qtype qtype,
984		unsigned int *pbar2_qid, u64 *pbar2_pa);
985extern struct cxgb4_client t4c_client;
986extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
987extern int c4iw_max_read_depth;
988
989#if defined(__i386__) || defined(__amd64__)
990#define L1_CACHE_BYTES 128
991#else
992#define L1_CACHE_BYTES 32
993#endif
994
995void your_reg_device(struct c4iw_dev *dev);
996
997#define SGE_CTRLQ_NUM	0
998
999#endif
1000