1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2020 Marvell. */
3
4#include "otx2_cptvf.h"
5#include "otx2_cpt_common.h"
6
7/* Default timeout when waiting for free pending entry in us */
8#define CPT_PENTRY_TIMEOUT	1000
9#define CPT_PENTRY_STEP		50
10
11/* Default threshold for stopping and resuming sender requests */
12#define CPT_IQ_STOP_MARGIN	128
13#define CPT_IQ_RESUME_MARGIN	512
14
15/* Default command timeout in seconds */
16#define CPT_COMMAND_TIMEOUT	4
17#define CPT_TIME_IN_RESET_COUNT 5
18
19static void otx2_cpt_dump_sg_list(struct pci_dev *pdev,
20				  struct otx2_cpt_req_info *req)
21{
22	int i;
23
24	pr_debug("Gather list size %d\n", req->in_cnt);
25	for (i = 0; i < req->in_cnt; i++) {
26		pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,
27			 req->in[i].size, req->in[i].vptr,
28			 req->in[i].dma_addr);
29		pr_debug("Buffer hexdump (%d bytes)\n",
30			 req->in[i].size);
31		print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
32				     req->in[i].vptr, req->in[i].size, false);
33	}
34	pr_debug("Scatter list size %d\n", req->out_cnt);
35	for (i = 0; i < req->out_cnt; i++) {
36		pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,
37			 req->out[i].size, req->out[i].vptr,
38			 req->out[i].dma_addr);
39		pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
40		print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
41				     req->out[i].vptr, req->out[i].size, false);
42	}
43}
44
45static inline struct otx2_cpt_pending_entry *get_free_pending_entry(
46					struct otx2_cpt_pending_queue *q,
47					int qlen)
48{
49	struct otx2_cpt_pending_entry *ent = NULL;
50
51	ent = &q->head[q->rear];
52	if (unlikely(ent->busy))
53		return NULL;
54
55	q->rear++;
56	if (unlikely(q->rear == qlen))
57		q->rear = 0;
58
59	return ent;
60}
61
62static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
63{
64	if (WARN_ON(inc > length))
65		inc = length;
66
67	index += inc;
68	if (unlikely(index >= length))
69		index -= length;
70
71	return index;
72}
73
74static inline void free_pentry(struct otx2_cpt_pending_entry *pentry)
75{
76	pentry->completion_addr = NULL;
77	pentry->info = NULL;
78	pentry->callback = NULL;
79	pentry->areq = NULL;
80	pentry->resume_sender = false;
81	pentry->busy = false;
82}
83
84static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
85			   struct otx2_cpt_pending_queue *pqueue,
86			   struct otx2_cptlf_info *lf)
87{
88	struct otx2_cptvf_request *cpt_req = &req->req;
89	struct otx2_cpt_pending_entry *pentry = NULL;
90	union otx2_cpt_ctrl_info *ctrl = &req->ctrl;
91	struct otx2_cpt_inst_info *info = NULL;
92	union otx2_cpt_res_s *result = NULL;
93	struct otx2_cpt_iq_command iq_cmd;
94	union otx2_cpt_inst_s cptinst;
95	int retry, ret = 0;
96	u8 resume_sender;
97	gfp_t gfp;
98
99	gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
100							      GFP_ATOMIC;
101	if (unlikely(!otx2_cptlf_started(lf->lfs)))
102		return -ENODEV;
103
104	info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp);
105	if (unlikely(!info)) {
106		dev_err(&pdev->dev, "Setting up cpt inst info failed");
107		return -ENOMEM;
108	}
109	cpt_req->dlen = info->dlen;
110
111	result = info->completion_addr;
112	result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
113
114	spin_lock_bh(&pqueue->lock);
115	pentry = get_free_pending_entry(pqueue, pqueue->qlen);
116	retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
117	while (unlikely(!pentry) && retry--) {
118		spin_unlock_bh(&pqueue->lock);
119		udelay(CPT_PENTRY_STEP);
120		spin_lock_bh(&pqueue->lock);
121		pentry = get_free_pending_entry(pqueue, pqueue->qlen);
122	}
123
124	if (unlikely(!pentry)) {
125		ret = -ENOSPC;
126		goto destroy_info;
127	}
128
129	/*
130	 * Check if we are close to filling in entire pending queue,
131	 * if so then tell the sender to stop/sleep by returning -EBUSY
132	 * We do it only for context which can sleep (GFP_KERNEL)
133	 */
134	if (gfp == GFP_KERNEL &&
135	    pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
136		pentry->resume_sender = true;
137	} else
138		pentry->resume_sender = false;
139	resume_sender = pentry->resume_sender;
140	pqueue->pending_count++;
141
142	pentry->completion_addr = info->completion_addr;
143	pentry->info = info;
144	pentry->callback = req->callback;
145	pentry->areq = req->areq;
146	pentry->busy = true;
147	info->pentry = pentry;
148	info->time_in = jiffies;
149	info->req = req;
150
151	/* Fill in the command */
152	iq_cmd.cmd.u = 0;
153	iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
154	iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
155	iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
156	iq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
157
158	/* 64-bit swap for microcode data reads, not needed for addresses*/
159	cpu_to_be64s(&iq_cmd.cmd.u);
160	iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60;
161	iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60;
162	iq_cmd.cptr.s.cptr = cpt_req->cptr_dma;
163	iq_cmd.cptr.s.grp = ctrl->s.grp;
164
165	/* Fill in the CPT_INST_S type command for HW interpretation */
166	otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr);
167
168	/* Print debug info if enabled */
169	otx2_cpt_dump_sg_list(pdev, req);
170	pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE);
171	print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false);
172	pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
173	print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
174			     cpt_req->dlen, false);
175
176	/* Send CPT command */
177	lf->lfs->ops->send_cmd(&cptinst, 1, lf);
178
179	/*
180	 * We allocate and prepare pending queue entry in critical section
181	 * together with submitting CPT instruction to CPT instruction queue
182	 * to make sure that order of CPT requests is the same in both
183	 * pending and instruction queues
184	 */
185	spin_unlock_bh(&pqueue->lock);
186
187	ret = resume_sender ? -EBUSY : -EINPROGRESS;
188	return ret;
189
190destroy_info:
191	spin_unlock_bh(&pqueue->lock);
192	otx2_cpt_info_destroy(pdev, info);
193	return ret;
194}
195
196int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
197			int cpu_num)
198{
199	struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
200	struct otx2_cptlfs_info *lfs = &cptvf->lfs;
201
202	return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue,
203			       &lfs->lf[cpu_num]);
204}
205
206static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,
207			     union otx2_cpt_res_s *cpt_status,
208			     struct otx2_cpt_inst_info *info,
209			     u32 *res_code)
210{
211	u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);
212	u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);
213	struct pci_dev *pdev = lfs->pdev;
214
215	switch (ccode) {
216	case OTX2_CPT_COMP_E_FAULT:
217		dev_err(&pdev->dev,
218			"Request failed with DMA fault\n");
219		otx2_cpt_dump_sg_list(pdev, info->req);
220		break;
221
222	case OTX2_CPT_COMP_E_HWERR:
223		dev_err(&pdev->dev,
224			"Request failed with hardware error\n");
225		otx2_cpt_dump_sg_list(pdev, info->req);
226		break;
227
228	case OTX2_CPT_COMP_E_INSTERR:
229		dev_err(&pdev->dev,
230			"Request failed with instruction error\n");
231		otx2_cpt_dump_sg_list(pdev, info->req);
232		break;
233
234	case OTX2_CPT_COMP_E_NOTDONE:
235		/* check for timeout */
236		if (time_after_eq(jiffies, info->time_in +
237				  CPT_COMMAND_TIMEOUT * HZ))
238			dev_warn(&pdev->dev,
239				 "Request timed out 0x%p", info->req);
240		else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) {
241			info->time_in = jiffies;
242			info->extra_time++;
243		}
244		return 1;
245
246	case OTX2_CPT_COMP_E_GOOD:
247	case OTX2_CPT_COMP_E_WARN:
248		/*
249		 * Check microcode completion code, it is only valid
250		 * when completion code is CPT_COMP_E::GOOD
251		 */
252		if (uc_ccode != OTX2_CPT_UCC_SUCCESS) {
253			/*
254			 * If requested hmac is truncated and ucode returns
255			 * s/g write length error then we report success
256			 * because ucode writes as many bytes of calculated
257			 * hmac as available in gather buffer and reports
258			 * s/g write length error if number of bytes in gather
259			 * buffer is less than full hmac size.
260			 */
261			if (info->req->is_trunc_hmac &&
262			    uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) {
263				*res_code = 0;
264				break;
265			}
266
267			dev_err(&pdev->dev,
268				"Request failed with software error code 0x%x\n",
269				cpt_status->s.uc_compcode);
270			otx2_cpt_dump_sg_list(pdev, info->req);
271			break;
272		}
273		/* Request has been processed with success */
274		*res_code = 0;
275		break;
276
277	default:
278		dev_err(&pdev->dev,
279			"Request returned invalid status %d\n", ccode);
280		break;
281	}
282	return 0;
283}
284
285static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,
286					 struct otx2_cpt_pending_queue *pqueue)
287{
288	struct otx2_cpt_pending_entry *resume_pentry = NULL;
289	void (*callback)(int status, void *arg, void *req);
290	struct otx2_cpt_pending_entry *pentry = NULL;
291	union otx2_cpt_res_s *cpt_status = NULL;
292	struct otx2_cpt_inst_info *info = NULL;
293	struct otx2_cpt_req_info *req = NULL;
294	struct crypto_async_request *areq;
295	struct pci_dev *pdev = lfs->pdev;
296	u32 res_code, resume_index;
297
298	while (1) {
299		spin_lock_bh(&pqueue->lock);
300		pentry = &pqueue->head[pqueue->front];
301
302		if (WARN_ON(!pentry)) {
303			spin_unlock_bh(&pqueue->lock);
304			break;
305		}
306
307		res_code = -EINVAL;
308		if (unlikely(!pentry->busy)) {
309			spin_unlock_bh(&pqueue->lock);
310			break;
311		}
312
313		if (unlikely(!pentry->callback)) {
314			dev_err(&pdev->dev, "Callback NULL\n");
315			goto process_pentry;
316		}
317
318		info = pentry->info;
319		if (unlikely(!info)) {
320			dev_err(&pdev->dev, "Pending entry post arg NULL\n");
321			goto process_pentry;
322		}
323
324		req = info->req;
325		if (unlikely(!req)) {
326			dev_err(&pdev->dev, "Request NULL\n");
327			goto process_pentry;
328		}
329
330		cpt_status = pentry->completion_addr;
331		if (unlikely(!cpt_status)) {
332			dev_err(&pdev->dev, "Completion address NULL\n");
333			goto process_pentry;
334		}
335
336		if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {
337			spin_unlock_bh(&pqueue->lock);
338			return;
339		}
340		info->pdev = pdev;
341
342process_pentry:
343		/*
344		 * Check if we should inform sending side to resume
345		 * We do it CPT_IQ_RESUME_MARGIN elements in advance before
346		 * pending queue becomes empty
347		 */
348		resume_index = modulo_inc(pqueue->front, pqueue->qlen,
349					  CPT_IQ_RESUME_MARGIN);
350		resume_pentry = &pqueue->head[resume_index];
351		if (resume_pentry &&
352		    resume_pentry->resume_sender) {
353			resume_pentry->resume_sender = false;
354			callback = resume_pentry->callback;
355			areq = resume_pentry->areq;
356
357			if (callback) {
358				spin_unlock_bh(&pqueue->lock);
359
360				/*
361				 * EINPROGRESS is an indication for sending
362				 * side that it can resume sending requests
363				 */
364				callback(-EINPROGRESS, areq, info);
365				spin_lock_bh(&pqueue->lock);
366			}
367		}
368
369		callback = pentry->callback;
370		areq = pentry->areq;
371		free_pentry(pentry);
372
373		pqueue->pending_count--;
374		pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
375		spin_unlock_bh(&pqueue->lock);
376
377		/*
378		 * Call callback after current pending entry has been
379		 * processed, we don't do it if the callback pointer is
380		 * invalid.
381		 */
382		if (callback)
383			callback(res_code, areq, info);
384	}
385}
386
387void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)
388{
389	process_pending_queue(wqe->lfs,
390			      &wqe->lfs->lf[wqe->lf_num].pqueue);
391}
392
393int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)
394{
395	struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
396
397	return cptvf->lfs.kcrypto_eng_grp_num;
398}
399