1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/socket.h>
3#include <linux/in.h>
4#include <linux/in6.h>
5#include <rdma/ib_verbs.h>
6#include <rdma/rdma_cm.h>
7#include <rdma/rw.h>
8#include <scsi/iser.h>
9
10
11#define DRV_NAME	"isert"
12#define PFX		DRV_NAME ": "
13
14#define isert_dbg(fmt, arg...)				 \
15	do {						 \
16		if (unlikely(isert_debug_level > 2))	 \
17			printk(KERN_DEBUG PFX "%s: " fmt,\
18				__func__ , ## arg);	 \
19	} while (0)
20
21#define isert_warn(fmt, arg...)				\
22	do {						\
23		if (unlikely(isert_debug_level > 0))	\
24			pr_warn(PFX "%s: " fmt,         \
25				__func__ , ## arg);	\
26	} while (0)
27
28#define isert_info(fmt, arg...)				\
29	do {						\
30		if (unlikely(isert_debug_level > 1))	\
31			pr_info(PFX "%s: " fmt,         \
32				__func__ , ## arg);	\
33	} while (0)
34
35#define isert_err(fmt, arg...) \
36	pr_err(PFX "%s: " fmt, __func__ , ## arg)
37
38/* Constant PDU lengths calculations */
39#define ISER_HEADERS_LEN	(sizeof(struct iser_ctrl) + \
40				 sizeof(struct iscsi_hdr))
41#define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
42
43/* QP settings */
44/* Maximal bounds on received asynchronous PDUs */
45#define ISERT_MAX_TX_MISC_PDUS	4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
46
47#define ISERT_MAX_RX_MISC_PDUS	6 /*
48				   * NOOP_OUT(2), TEXT(1),
49				   * SCSI_TMFUNC(2), LOGOUT(1)
50				   */
51
52#define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */
53
54#define ISERT_QP_MAX_RECV_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX)
55
56#define ISERT_MIN_POSTED_RX	(ISCSI_DEF_XMIT_CMDS_MAX >> 2)
57
58#define ISERT_QP_MAX_REQ_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX +    \
59				ISERT_MAX_TX_MISC_PDUS	+ \
60				ISERT_MAX_RX_MISC_PDUS)
61
62/*
63 * RX size is default of 8k plus headers, but data needs to align to
64 * 512 boundary, so use 1024 to have the extra space for alignment.
65 */
66#define ISER_RX_SIZE		(ISCSI_DEF_MAX_RECV_SEG_LEN + 1024)
67
68/* Minimum I/O size is 512KB */
69#define ISCSI_ISER_MIN_SG_TABLESIZE 128
70
71/* Maximum support is 16MB I/O size */
72#define ISCSI_ISER_MAX_SG_TABLESIZE	4096
73
74enum isert_desc_type {
75	ISCSI_TX_CONTROL,
76	ISCSI_TX_DATAIN
77};
78
79enum iser_conn_state {
80	ISER_CONN_INIT,
81	ISER_CONN_UP,
82	ISER_CONN_BOUND,
83	ISER_CONN_FULL_FEATURE,
84	ISER_CONN_TERMINATING,
85	ISER_CONN_DOWN,
86};
87
88struct iser_rx_desc {
89	char		buf[ISER_RX_SIZE];
90	u64		dma_addr;
91	struct ib_sge	rx_sg;
92	struct ib_cqe	rx_cqe;
93	bool		in_use;
94};
95
96static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe)
97{
98	return container_of(cqe, struct iser_rx_desc, rx_cqe);
99}
100
101static void *isert_get_iser_hdr(struct iser_rx_desc *desc)
102{
103	return PTR_ALIGN(desc->buf + ISER_HEADERS_LEN, 512) - ISER_HEADERS_LEN;
104}
105
106static size_t isert_get_hdr_offset(struct iser_rx_desc *desc)
107{
108	return isert_get_iser_hdr(desc) - (void *)desc->buf;
109}
110
111static void *isert_get_iscsi_hdr(struct iser_rx_desc *desc)
112{
113	return isert_get_iser_hdr(desc) + sizeof(struct iser_ctrl);
114}
115
116static void *isert_get_data(struct iser_rx_desc *desc)
117{
118	void *data = isert_get_iser_hdr(desc) + ISER_HEADERS_LEN;
119
120	WARN_ON((uintptr_t)data & 511);
121	return data;
122}
123
124struct iser_tx_desc {
125	struct iser_ctrl iser_header;
126	struct iscsi_hdr iscsi_header;
127	enum isert_desc_type type;
128	u64		dma_addr;
129	struct ib_sge	tx_sg[2];
130	struct ib_cqe	tx_cqe;
131	int		num_sge;
132	struct ib_send_wr send_wr;
133} __packed;
134
135static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe)
136{
137	return container_of(cqe, struct iser_tx_desc, tx_cqe);
138}
139
140struct isert_cmd {
141	uint32_t		read_stag;
142	uint32_t		write_stag;
143	uint64_t		read_va;
144	uint64_t		write_va;
145	uint32_t		inv_rkey;
146	u64			pdu_buf_dma;
147	u32			pdu_buf_len;
148	struct isert_conn	*conn;
149	struct iscsit_cmd	*iscsit_cmd;
150	struct iser_tx_desc	tx_desc;
151	struct iser_rx_desc	*rx_desc;
152	struct rdma_rw_ctx	rw;
153	struct work_struct	comp_work;
154	struct scatterlist	sg;
155	bool			ctx_init_done;
156};
157
158static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc)
159{
160	return container_of(desc, struct isert_cmd, tx_desc);
161}
162
163struct isert_device;
164
165struct isert_conn {
166	enum iser_conn_state	state;
167	u32			responder_resources;
168	u32			initiator_depth;
169	bool			pi_support;
170	struct iser_rx_desc	*login_desc;
171	char			*login_rsp_buf;
172	int			login_req_len;
173	u64			login_rsp_dma;
174	struct iser_rx_desc	*rx_descs;
175	struct ib_recv_wr	rx_wr[ISERT_QP_MAX_RECV_DTOS];
176	struct iscsit_conn	*conn;
177	struct list_head	node;
178	struct completion	login_comp;
179	struct completion	login_req_comp;
180	struct iser_tx_desc	login_tx_desc;
181	struct rdma_cm_id	*cm_id;
182	struct ib_qp		*qp;
183	struct ib_cq		*cq;
184	u32			cq_size;
185	struct isert_device	*device;
186	struct mutex		mutex;
187	struct kref		kref;
188	struct work_struct	release_work;
189	bool                    logout_posted;
190	bool                    snd_w_inv;
191	wait_queue_head_t	rem_wait;
192	bool			dev_removed;
193};
194
195struct isert_device {
196	bool			pi_capable;
197	int			refcount;
198	struct ib_device	*ib_device;
199	struct ib_pd		*pd;
200	struct isert_comp	*comps;
201	int                     comps_used;
202	struct list_head	dev_node;
203};
204
205struct isert_np {
206	struct iscsi_np         *np;
207	struct semaphore	sem;
208	struct rdma_cm_id	*cm_id;
209	struct mutex		mutex;
210	struct list_head	accepted;
211	struct list_head	pending;
212};
213