1// SPDX-License-Identifier: GPL-2.0
2#include <linux/gfp.h>
3#include <linux/workqueue.h>
4#include <crypto/internal/skcipher.h>
5
6#include "nitrox_common.h"
7#include "nitrox_dev.h"
8#include "nitrox_req.h"
9#include "nitrox_csr.h"
10
11/* SLC_STORE_INFO */
12#define MIN_UDD_LEN 16
13/* PKT_IN_HDR + SLC_STORE_INFO */
14#define FDATA_SIZE 32
15/* Base destination port for the solicited requests */
16#define SOLICIT_BASE_DPORT 256
17
18#define REQ_NOT_POSTED 1
19#define REQ_BACKLOG    2
20#define REQ_POSTED     3
21
22/*
23 * Response codes from SE microcode
24 * 0x00 - Success
25 *   Completion with no error
26 * 0x43 - ERR_GC_DATA_LEN_INVALID
27 *   Invalid Data length if Encryption Data length is
28 *   less than 16 bytes for AES-XTS and AES-CTS.
29 * 0x45 - ERR_GC_CTX_LEN_INVALID
30 *   Invalid context length: CTXL != 23 words.
31 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
32 *   DOCSIS support is enabled with other than
33 *   AES/DES-CBC mode encryption.
34 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
35 *   Authentication offset is other than 0 with
36 *   Encryption IV source = 0.
37 *   Authentication offset is other than 8 (DES)/16 (AES)
38 *   with Encryption IV source = 1
39 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
40 *   CRC32 is enabled for other than DOCSIS encryption.
41 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
42 *   Invalid flag options in AES-CCM IV.
43 */
44
45static inline int incr_index(int index, int count, int max)
46{
47	if ((index + count) >= max)
48		index = index + count - max;
49	else
50		index += count;
51
52	return index;
53}
54
55static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
56{
57	struct nitrox_device *ndev = sr->ndev;
58	struct device *dev = DEV(ndev);
59
60
61	dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg),
62		     DMA_BIDIRECTIONAL);
63	dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
64			 DMA_TO_DEVICE);
65	kfree(sr->in.sgcomp);
66	sr->in.sg = NULL;
67	sr->in.sgmap_cnt = 0;
68
69	dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg),
70		     DMA_BIDIRECTIONAL);
71	dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
72			 DMA_TO_DEVICE);
73	kfree(sr->out.sgcomp);
74	sr->out.sg = NULL;
75	sr->out.sgmap_cnt = 0;
76}
77
78static void softreq_destroy(struct nitrox_softreq *sr)
79{
80	softreq_unmap_sgbufs(sr);
81	kfree(sr);
82}
83
84/**
85 * create_sg_component - create SG componets for N5 device.
86 * @sr: Request structure
87 * @sgtbl: SG table
88 * @map_nents: number of dma mapped entries
89 *
90 * Component structure
91 *
92 *   63     48 47     32 31    16 15      0
93 *   --------------------------------------
94 *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
95 *   |-------------------------------------
96 *   |               PTR0                 |
97 *   --------------------------------------
98 *   |               PTR1                 |
99 *   --------------------------------------
100 *   |               PTR2                 |
101 *   --------------------------------------
102 *   |               PTR3                 |
103 *   --------------------------------------
104 *
105 *   Returns 0 if success or a negative errno code on error.
106 */
107static int create_sg_component(struct nitrox_softreq *sr,
108			       struct nitrox_sgtable *sgtbl, int map_nents)
109{
110	struct nitrox_device *ndev = sr->ndev;
111	struct nitrox_sgcomp *sgcomp;
112	struct scatterlist *sg;
113	dma_addr_t dma;
114	size_t sz_comp;
115	int i, j, nr_sgcomp;
116
117	nr_sgcomp = roundup(map_nents, 4) / 4;
118
119	/* each component holds 4 dma pointers */
120	sz_comp = nr_sgcomp * sizeof(*sgcomp);
121	sgcomp = kzalloc(sz_comp, sr->gfp);
122	if (!sgcomp)
123		return -ENOMEM;
124
125	sgtbl->sgcomp = sgcomp;
126
127	sg = sgtbl->sg;
128	/* populate device sg component */
129	for (i = 0; i < nr_sgcomp; i++) {
130		for (j = 0; j < 4 && sg; j++) {
131			sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
132			sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
133			sg = sg_next(sg);
134		}
135	}
136	/* map the device sg component */
137	dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
138	if (dma_mapping_error(DEV(ndev), dma)) {
139		kfree(sgtbl->sgcomp);
140		sgtbl->sgcomp = NULL;
141		return -ENOMEM;
142	}
143
144	sgtbl->sgcomp_dma = dma;
145	sgtbl->sgcomp_len = sz_comp;
146
147	return 0;
148}
149
150/**
151 * dma_map_inbufs - DMA map input sglist and creates sglist component
152 *                  for N5 device.
153 * @sr: Request structure
154 * @req: Crypto request structre
155 *
156 * Returns 0 if successful or a negative errno code on error.
157 */
158static int dma_map_inbufs(struct nitrox_softreq *sr,
159			  struct se_crypto_request *req)
160{
161	struct device *dev = DEV(sr->ndev);
162	struct scatterlist *sg;
163	int i, nents, ret = 0;
164
165	nents = dma_map_sg(dev, req->src, sg_nents(req->src),
166			   DMA_BIDIRECTIONAL);
167	if (!nents)
168		return -EINVAL;
169
170	for_each_sg(req->src, sg, nents, i)
171		sr->in.total_bytes += sg_dma_len(sg);
172
173	sr->in.sg = req->src;
174	sr->in.sgmap_cnt = nents;
175	ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
176	if (ret)
177		goto incomp_err;
178
179	return 0;
180
181incomp_err:
182	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL);
183	sr->in.sgmap_cnt = 0;
184	return ret;
185}
186
187static int dma_map_outbufs(struct nitrox_softreq *sr,
188			   struct se_crypto_request *req)
189{
190	struct device *dev = DEV(sr->ndev);
191	int nents, ret = 0;
192
193	nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
194			   DMA_BIDIRECTIONAL);
195	if (!nents)
196		return -EINVAL;
197
198	sr->out.sg = req->dst;
199	sr->out.sgmap_cnt = nents;
200	ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
201	if (ret)
202		goto outcomp_map_err;
203
204	return 0;
205
206outcomp_map_err:
207	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL);
208	sr->out.sgmap_cnt = 0;
209	sr->out.sg = NULL;
210	return ret;
211}
212
213static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
214				    struct se_crypto_request *creq)
215{
216	int ret;
217
218	ret = dma_map_inbufs(sr, creq);
219	if (ret)
220		return ret;
221
222	ret = dma_map_outbufs(sr, creq);
223	if (ret)
224		softreq_unmap_sgbufs(sr);
225
226	return ret;
227}
228
229static inline void backlog_list_add(struct nitrox_softreq *sr,
230				    struct nitrox_cmdq *cmdq)
231{
232	INIT_LIST_HEAD(&sr->backlog);
233
234	spin_lock_bh(&cmdq->backlog_qlock);
235	list_add_tail(&sr->backlog, &cmdq->backlog_head);
236	atomic_inc(&cmdq->backlog_count);
237	atomic_set(&sr->status, REQ_BACKLOG);
238	spin_unlock_bh(&cmdq->backlog_qlock);
239}
240
241static inline void response_list_add(struct nitrox_softreq *sr,
242				     struct nitrox_cmdq *cmdq)
243{
244	INIT_LIST_HEAD(&sr->response);
245
246	spin_lock_bh(&cmdq->resp_qlock);
247	list_add_tail(&sr->response, &cmdq->response_head);
248	spin_unlock_bh(&cmdq->resp_qlock);
249}
250
251static inline void response_list_del(struct nitrox_softreq *sr,
252				     struct nitrox_cmdq *cmdq)
253{
254	spin_lock_bh(&cmdq->resp_qlock);
255	list_del(&sr->response);
256	spin_unlock_bh(&cmdq->resp_qlock);
257}
258
259static struct nitrox_softreq *
260get_first_response_entry(struct nitrox_cmdq *cmdq)
261{
262	return list_first_entry_or_null(&cmdq->response_head,
263					struct nitrox_softreq, response);
264}
265
266static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
267{
268	if (atomic_inc_return(&cmdq->pending_count) > qlen) {
269		atomic_dec(&cmdq->pending_count);
270		/* sync with other cpus */
271		smp_mb__after_atomic();
272		return true;
273	}
274	/* sync with other cpus */
275	smp_mb__after_atomic();
276	return false;
277}
278
279/**
280 * post_se_instr - Post SE instruction to Packet Input ring
281 * @sr: Request structure
282 * @cmdq: Command queue structure
283 *
284 * Returns 0 if successful or a negative error code,
285 * if no space in ring.
286 */
287static void post_se_instr(struct nitrox_softreq *sr,
288			  struct nitrox_cmdq *cmdq)
289{
290	struct nitrox_device *ndev = sr->ndev;
291	int idx;
292	u8 *ent;
293
294	spin_lock_bh(&cmdq->cmd_qlock);
295
296	idx = cmdq->write_idx;
297	/* copy the instruction */
298	ent = cmdq->base + (idx * cmdq->instr_size);
299	memcpy(ent, &sr->instr, cmdq->instr_size);
300
301	atomic_set(&sr->status, REQ_POSTED);
302	response_list_add(sr, cmdq);
303	sr->tstamp = jiffies;
304	/* flush the command queue updates */
305	dma_wmb();
306
307	/* Ring doorbell with count 1 */
308	writeq(1, cmdq->dbell_csr_addr);
309
310	cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
311
312	spin_unlock_bh(&cmdq->cmd_qlock);
313
314	/* increment the posted command count */
315	atomic64_inc(&ndev->stats.posted);
316}
317
318static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
319{
320	struct nitrox_device *ndev = cmdq->ndev;
321	struct nitrox_softreq *sr, *tmp;
322	int ret = 0;
323
324	if (!atomic_read(&cmdq->backlog_count))
325		return 0;
326
327	spin_lock_bh(&cmdq->backlog_qlock);
328
329	list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
330		/* submit until space available */
331		if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
332			ret = -ENOSPC;
333			break;
334		}
335		/* delete from backlog list */
336		list_del(&sr->backlog);
337		atomic_dec(&cmdq->backlog_count);
338		/* sync with other cpus */
339		smp_mb__after_atomic();
340
341		/* post the command */
342		post_se_instr(sr, cmdq);
343	}
344	spin_unlock_bh(&cmdq->backlog_qlock);
345
346	return ret;
347}
348
349static int nitrox_enqueue_request(struct nitrox_softreq *sr)
350{
351	struct nitrox_cmdq *cmdq = sr->cmdq;
352	struct nitrox_device *ndev = sr->ndev;
353
354	/* try to post backlog requests */
355	post_backlog_cmds(cmdq);
356
357	if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
358		if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
359			/* increment drop count */
360			atomic64_inc(&ndev->stats.dropped);
361			return -ENOSPC;
362		}
363		/* add to backlog list */
364		backlog_list_add(sr, cmdq);
365		return -EINPROGRESS;
366	}
367	post_se_instr(sr, cmdq);
368
369	return -EINPROGRESS;
370}
371
372/**
373 * nitrox_process_se_request - Send request to SE core
374 * @ndev: NITROX device
375 * @req: Crypto request
376 * @callback: Completion callback
377 * @cb_arg: Completion callback arguments
378 *
379 * Returns 0 on success, or a negative error code.
380 */
381int nitrox_process_se_request(struct nitrox_device *ndev,
382			      struct se_crypto_request *req,
383			      completion_t callback,
384			      void *cb_arg)
385{
386	struct nitrox_softreq *sr;
387	dma_addr_t ctx_handle = 0;
388	int qno, ret = 0;
389
390	if (!nitrox_ready(ndev))
391		return -ENODEV;
392
393	sr = kzalloc(sizeof(*sr), req->gfp);
394	if (!sr)
395		return -ENOMEM;
396
397	sr->ndev = ndev;
398	sr->flags = req->flags;
399	sr->gfp = req->gfp;
400	sr->callback = callback;
401	sr->cb_arg = cb_arg;
402
403	atomic_set(&sr->status, REQ_NOT_POSTED);
404
405	sr->resp.orh = req->orh;
406	sr->resp.completion = req->comp;
407
408	ret = softreq_map_iobuf(sr, req);
409	if (ret) {
410		kfree(sr);
411		return ret;
412	}
413
414	/* get the context handle */
415	if (req->ctx_handle) {
416		struct ctx_hdr *hdr;
417		u8 *ctx_ptr;
418
419		ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
420		hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
421		ctx_handle = hdr->ctx_dma;
422	}
423
424	/* select the queue */
425	qno = smp_processor_id() % ndev->nr_queues;
426
427	sr->cmdq = &ndev->pkt_inq[qno];
428
429	/*
430	 * 64-Byte Instruction Format
431	 *
432	 *  ----------------------
433	 *  |      DPTR0         | 8 bytes
434	 *  ----------------------
435	 *  |  PKT_IN_INSTR_HDR  | 8 bytes
436	 *  ----------------------
437	 *  |    PKT_IN_HDR      | 16 bytes
438	 *  ----------------------
439	 *  |    SLC_INFO        | 16 bytes
440	 *  ----------------------
441	 *  |   Front data       | 16 bytes
442	 *  ----------------------
443	 */
444
445	/* fill the packet instruction */
446	/* word 0 */
447	sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
448
449	/* word 1 */
450	sr->instr.ih.value = 0;
451	sr->instr.ih.s.g = 1;
452	sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
453	sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
454	sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
455	sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
456	sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value);
457
458	/* word 2 */
459	sr->instr.irh.value[0] = 0;
460	sr->instr.irh.s.uddl = MIN_UDD_LEN;
461	/* context length in 64-bit words */
462	sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
463	/* offset from solicit base port 256 */
464	sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
465	sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
466	sr->instr.irh.s.arg = req->ctrl.s.arg;
467	sr->instr.irh.s.opcode = req->opcode;
468	sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]);
469
470	/* word 3 */
471	sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
472
473	/* word 4 */
474	sr->instr.slc.value[0] = 0;
475	sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
476	sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]);
477
478	/* word 5 */
479	sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
480
481	/*
482	 * No conversion for front data,
483	 * It goes into payload
484	 * put GP Header in front data
485	 */
486	sr->instr.fdata[0] = *((u64 *)&req->gph);
487	sr->instr.fdata[1] = 0;
488
489	ret = nitrox_enqueue_request(sr);
490	if (ret == -ENOSPC)
491		goto send_fail;
492
493	return ret;
494
495send_fail:
496	softreq_destroy(sr);
497	return ret;
498}
499
500static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
501{
502	return time_after_eq(jiffies, (tstamp + timeout));
503}
504
505void backlog_qflush_work(struct work_struct *work)
506{
507	struct nitrox_cmdq *cmdq;
508
509	cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
510	post_backlog_cmds(cmdq);
511}
512
513static bool sr_completed(struct nitrox_softreq *sr)
514{
515	u64 orh = READ_ONCE(*sr->resp.orh);
516	unsigned long timeout = jiffies + msecs_to_jiffies(1);
517
518	if ((orh != PENDING_SIG) && (orh & 0xff))
519		return true;
520
521	while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
522		if (time_after(jiffies, timeout)) {
523			pr_err("comp not done\n");
524			return false;
525		}
526	}
527
528	return true;
529}
530
531/**
532 * process_response_list - process completed requests
533 * @cmdq: Command queue structure
534 *
535 * Returns the number of responses processed.
536 */
537static void process_response_list(struct nitrox_cmdq *cmdq)
538{
539	struct nitrox_device *ndev = cmdq->ndev;
540	struct nitrox_softreq *sr;
541	int req_completed = 0, err = 0, budget;
542	completion_t callback;
543	void *cb_arg;
544
545	/* check all pending requests */
546	budget = atomic_read(&cmdq->pending_count);
547
548	while (req_completed < budget) {
549		sr = get_first_response_entry(cmdq);
550		if (!sr)
551			break;
552
553		if (atomic_read(&sr->status) != REQ_POSTED)
554			break;
555
556		/* check orh and completion bytes updates */
557		if (!sr_completed(sr)) {
558			/* request not completed, check for timeout */
559			if (!cmd_timeout(sr->tstamp, ndev->timeout))
560				break;
561			dev_err_ratelimited(DEV(ndev),
562					    "Request timeout, orh 0x%016llx\n",
563					    READ_ONCE(*sr->resp.orh));
564		}
565		atomic_dec(&cmdq->pending_count);
566		atomic64_inc(&ndev->stats.completed);
567		/* sync with other cpus */
568		smp_mb__after_atomic();
569		/* remove from response list */
570		response_list_del(sr, cmdq);
571		/* ORH error code */
572		err = READ_ONCE(*sr->resp.orh) & 0xff;
573		callback = sr->callback;
574		cb_arg = sr->cb_arg;
575		softreq_destroy(sr);
576		if (callback)
577			callback(cb_arg, err);
578
579		req_completed++;
580	}
581}
582
583/*
584 * pkt_slc_resp_tasklet - post processing of SE responses
585 */
586void pkt_slc_resp_tasklet(unsigned long data)
587{
588	struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
589	struct nitrox_cmdq *cmdq = qvec->cmdq;
590	union nps_pkt_slc_cnts slc_cnts;
591
592	/* read completion count */
593	slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
594	/* resend the interrupt if more work to do */
595	slc_cnts.s.resend = 1;
596
597	process_response_list(cmdq);
598
599	/*
600	 * clear the interrupt with resend bit enabled,
601	 * MSI-X interrupt generates if Completion count > Threshold
602	 */
603	writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
604
605	if (atomic_read(&cmdq->backlog_count))
606		schedule_work(&cmdq->backlog_qflush);
607}
608