1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2016 Chelsio Communications, Inc.
4 */
5
6#include "cxgbit.h"
7
8static void
9cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod,
10		    struct cxgbi_task_tag_info *ttinfo,
11		    struct scatterlist **sg_pp, unsigned int *sg_off)
12{
13	struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
14	unsigned int offset = sg_off ? *sg_off : 0;
15	dma_addr_t addr = 0UL;
16	unsigned int len = 0;
17	int i;
18
19	memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
20
21	if (sg) {
22		addr = sg_dma_address(sg);
23		len = sg_dma_len(sg);
24	}
25
26	for (i = 0; i < PPOD_PAGES_MAX; i++) {
27		if (sg) {
28			ppod->addr[i] = cpu_to_be64(addr + offset);
29			offset += PAGE_SIZE;
30			if (offset == (len + sg->offset)) {
31				offset = 0;
32				sg = sg_next(sg);
33				if (sg) {
34					addr = sg_dma_address(sg);
35					len = sg_dma_len(sg);
36				}
37			}
38		} else {
39			ppod->addr[i] = 0ULL;
40		}
41	}
42
43	/*
44	 * the fifth address needs to be repeated in the next ppod, so do
45	 * not move sg
46	 */
47	if (sg_pp) {
48		*sg_pp = sg;
49		*sg_off = offset;
50	}
51
52	if (offset == len) {
53		offset = 0;
54		if (sg) {
55			sg = sg_next(sg);
56			if (sg)
57				addr = sg_dma_address(sg);
58		}
59	}
60	ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
61}
62
63static struct sk_buff *
64cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm,
65		       unsigned int idx, unsigned int npods, unsigned int tid)
66{
67	struct ulp_mem_io *req;
68	struct ulptx_idata *idata;
69	unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
70	unsigned int dlen = npods << PPOD_SIZE_SHIFT;
71	unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
72				sizeof(struct ulptx_idata) + dlen, 16);
73	struct sk_buff *skb;
74
75	skb  = alloc_skb(wr_len, GFP_KERNEL);
76	if (!skb)
77		return NULL;
78
79	req = __skb_put(skb, wr_len);
80	INIT_ULPTX_WR(req, wr_len, 0, tid);
81	req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) |
82		FW_WR_ATOMIC_V(0));
83	req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
84		ULP_MEMIO_ORDER_V(0) |
85		T5_ULP_MEMIO_IMM_V(1));
86	req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
87	req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
88	req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
89
90	idata = (struct ulptx_idata *)(req + 1);
91	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
92	idata->len = htonl(dlen);
93
94	return skb;
95}
96
97static int
98cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
99			struct cxgbi_task_tag_info *ttinfo, unsigned int idx,
100			unsigned int npods, struct scatterlist **sg_pp,
101			unsigned int *sg_off)
102{
103	struct cxgbit_device *cdev = csk->com.cdev;
104	struct sk_buff *skb;
105	struct ulp_mem_io *req;
106	struct ulptx_idata *idata;
107	struct cxgbi_pagepod *ppod;
108	unsigned int i;
109
110	skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, csk->tid);
111	if (!skb)
112		return -ENOMEM;
113
114	req = (struct ulp_mem_io *)skb->data;
115	idata = (struct ulptx_idata *)(req + 1);
116	ppod = (struct cxgbi_pagepod *)(idata + 1);
117
118	for (i = 0; i < npods; i++, ppod++)
119		cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off);
120
121	__skb_queue_tail(&csk->ppodq, skb);
122
123	return 0;
124}
125
126static int
127cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
128		   struct cxgbi_task_tag_info *ttinfo)
129{
130	unsigned int pidx = ttinfo->idx;
131	unsigned int npods = ttinfo->npods;
132	unsigned int i, cnt;
133	struct scatterlist *sg = ttinfo->sgl;
134	unsigned int offset = 0;
135	int ret = 0;
136
137	for (i = 0; i < npods; i += cnt, pidx += cnt) {
138		cnt = npods - i;
139
140		if (cnt > ULPMEM_IDATA_MAX_NPPODS)
141			cnt = ULPMEM_IDATA_MAX_NPPODS;
142
143		ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt,
144					      &sg, &offset);
145		if (ret < 0)
146			break;
147	}
148
149	return ret;
150}
151
152static int cxgbit_ddp_sgl_check(struct scatterlist *sg,
153				unsigned int nents)
154{
155	unsigned int last_sgidx = nents - 1;
156	unsigned int i;
157
158	for (i = 0; i < nents; i++, sg = sg_next(sg)) {
159		unsigned int len = sg->length + sg->offset;
160
161		if ((sg->offset & 0x3) || (i && sg->offset) ||
162		    ((i != last_sgidx) && (len != PAGE_SIZE))) {
163			return -EINVAL;
164		}
165	}
166
167	return 0;
168}
169
170static int
171cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo,
172		   unsigned int xferlen)
173{
174	struct cxgbit_device *cdev = csk->com.cdev;
175	struct cxgbi_ppm *ppm = cdev2ppm(cdev);
176	struct scatterlist *sgl = ttinfo->sgl;
177	unsigned int sgcnt = ttinfo->nents;
178	unsigned int sg_offset = sgl->offset;
179	int ret;
180
181	if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) {
182		pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
183			 ppm, ppm->tformat.pgsz_idx_dflt,
184			 xferlen, ttinfo->nents);
185		return -EINVAL;
186	}
187
188	if (cxgbit_ddp_sgl_check(sgl, sgcnt) < 0)
189		return -EINVAL;
190
191	ttinfo->nr_pages = (xferlen + sgl->offset +
192			    (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT;
193
194	/*
195	 * the ddp tag will be used for the ttt in the outgoing r2t pdu
196	 */
197	ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
198				      &ttinfo->tag, 0);
199	if (ret < 0)
200		return ret;
201	ttinfo->npods = ret;
202
203	sgl->offset = 0;
204	ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
205	sgl->offset = sg_offset;
206	if (!ret) {
207		pr_debug("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
208			 __func__, 0, xferlen, sgcnt);
209		goto rel_ppods;
210	}
211
212	cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
213				xferlen, &ttinfo->hdr);
214
215	ret = cxgbit_ddp_set_map(ppm, csk, ttinfo);
216	if (ret < 0) {
217		__skb_queue_purge(&csk->ppodq);
218		dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
219		goto rel_ppods;
220	}
221
222	return 0;
223
224rel_ppods:
225	cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
226	return -EINVAL;
227}
228
229void
230cxgbit_get_r2t_ttt(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
231		   struct iscsi_r2t *r2t)
232{
233	struct cxgbit_sock *csk = conn->context;
234	struct cxgbit_device *cdev = csk->com.cdev;
235	struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
236	struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
237	int ret;
238
239	if ((!ccmd->setup_ddp) ||
240	    (!test_bit(CSK_DDP_ENABLE, &csk->com.flags)))
241		goto out;
242
243	ccmd->setup_ddp = false;
244
245	ttinfo->sgl = cmd->se_cmd.t_data_sg;
246	ttinfo->nents = cmd->se_cmd.t_data_nents;
247
248	ret = cxgbit_ddp_reserve(csk, ttinfo, cmd->se_cmd.data_length);
249	if (ret < 0) {
250		pr_debug("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n",
251			 csk, cmd, cmd->se_cmd.data_length, ttinfo->nents);
252
253		ttinfo->sgl = NULL;
254		ttinfo->nents = 0;
255	} else {
256		ccmd->release = true;
257	}
258out:
259	pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n", cdev, cmd, ttinfo->tag);
260	r2t->targ_xfer_tag = ttinfo->tag;
261}
262
263void cxgbit_unmap_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
264{
265	struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
266
267	if (ccmd->release) {
268		if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
269			put_page(sg_page(&ccmd->sg));
270		} else {
271			struct cxgbit_sock *csk = conn->context;
272			struct cxgbit_device *cdev = csk->com.cdev;
273			struct cxgbi_ppm *ppm = cdev2ppm(cdev);
274			struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
275
276			/* Abort the TCP conn if DDP is not complete to
277			 * avoid any possibility of DDP after freeing
278			 * the cmd.
279			 */
280			if (unlikely(cmd->write_data_done !=
281				     cmd->se_cmd.data_length))
282				cxgbit_abort_conn(csk);
283
284			if (unlikely(ttinfo->sgl)) {
285				dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl,
286					     ttinfo->nents, DMA_FROM_DEVICE);
287				ttinfo->nents = 0;
288				ttinfo->sgl = NULL;
289			}
290			cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
291		}
292		ccmd->release = false;
293	}
294}
295
296int cxgbit_ddp_init(struct cxgbit_device *cdev)
297{
298	struct cxgb4_lld_info *lldi = &cdev->lldi;
299	struct net_device *ndev = cdev->lldi.ports[0];
300	struct cxgbi_tag_format tformat;
301	int ret, i;
302
303	if (!lldi->vr->iscsi.size) {
304		pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name);
305		return -EACCES;
306	}
307
308	memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
309	for (i = 0; i < 4; i++)
310		tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3))
311					 & 0xF;
312	cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat);
313
314	ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0],
315			     cdev->lldi.pdev, &cdev->lldi, &tformat,
316			     lldi->vr->iscsi.size, lldi->iscsi_llimit,
317			     lldi->vr->iscsi.start, 2,
318			     lldi->vr->ppod_edram.start,
319			     lldi->vr->ppod_edram.size);
320	if (ret >= 0) {
321		struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm);
322
323		if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) &&
324		    (ppm->ppmax >= 1024))
325			set_bit(CDEV_DDP_ENABLE, &cdev->flags);
326		ret = 0;
327	}
328
329	return ret;
330}
331