1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __NITROX_REQ_H
3#define __NITROX_REQ_H
4
5#include <linux/dma-mapping.h>
6#include <crypto/aes.h>
7
8#include "nitrox_dev.h"
9
10#define PENDING_SIG	0xFFFFFFFFFFFFFFFFUL
11#define PRIO 4001
12
13typedef void (*sereq_completion_t)(void *req, int err);
14
15/**
16 * struct gphdr - General purpose Header
17 * @param0: first parameter.
18 * @param1: second parameter.
19 * @param2: third parameter.
20 * @param3: fourth parameter.
21 *
22 * Params tell the iv and enc/dec data offsets.
23 */
24struct gphdr {
25	__be16 param0;
26	__be16 param1;
27	__be16 param2;
28	__be16 param3;
29};
30
31/**
32 * struct se_req_ctrl - SE request information.
33 * @arg: Minor number of the opcode
34 * @ctxc: Context control.
35 * @unca: Uncertainity enabled.
36 * @info: Additional information for SE cores.
37 * @ctxl: Context length in bytes.
38 * @uddl: User defined data length
39 */
40union se_req_ctrl {
41	u64 value;
42	struct {
43		u64 raz	: 22;
44		u64 arg	: 8;
45		u64 ctxc : 2;
46		u64 unca : 1;
47		u64 info : 3;
48		u64 unc : 8;
49		u64 ctxl : 12;
50		u64 uddl : 8;
51	} s;
52};
53
54#define MAX_IV_LEN 16
55
56/**
57 * struct se_crypto_request - SE crypto request structure.
58 * @opcode: Request opcode (enc/dec)
59 * @flags: flags from crypto subsystem
60 * @ctx_handle: Crypto context handle.
61 * @gph: GP Header
62 * @ctrl: Request Information.
63 * @orh: ORH address
64 * @comp: completion address
65 * @src: Input sglist
66 * @dst: Output sglist
67 */
68struct se_crypto_request {
69	u8 opcode;
70	gfp_t gfp;
71	u32 flags;
72	u64 ctx_handle;
73
74	struct gphdr gph;
75	union se_req_ctrl ctrl;
76	u64 *orh;
77	u64 *comp;
78
79	struct scatterlist *src;
80	struct scatterlist *dst;
81};
82
83/* Crypto opcodes */
84#define FLEXI_CRYPTO_ENCRYPT_HMAC	0x33
85#define ENCRYPT	0
86#define DECRYPT 1
87
88/* IV from context */
89#define IV_FROM_CTX	0
90/* IV from Input data */
91#define IV_FROM_DPTR	1
92
93/**
94 * cipher opcodes for firmware
95 */
96enum flexi_cipher {
97	CIPHER_NULL = 0,
98	CIPHER_3DES_CBC,
99	CIPHER_3DES_ECB,
100	CIPHER_AES_CBC,
101	CIPHER_AES_ECB,
102	CIPHER_AES_CFB,
103	CIPHER_AES_CTR,
104	CIPHER_AES_GCM,
105	CIPHER_AES_XTS,
106	CIPHER_AES_CCM,
107	CIPHER_AES_CBC_CTS,
108	CIPHER_AES_ECB_CTS,
109	CIPHER_INVALID
110};
111
112enum flexi_auth {
113	AUTH_NULL = 0,
114	AUTH_MD5,
115	AUTH_SHA1,
116	AUTH_SHA2_SHA224,
117	AUTH_SHA2_SHA256,
118	AUTH_SHA2_SHA384,
119	AUTH_SHA2_SHA512,
120	AUTH_GMAC,
121	AUTH_INVALID
122};
123
124/**
125 * struct crypto_keys - Crypto keys
126 * @key: Encryption key or KEY1 for AES-XTS
127 * @iv: Encryption IV or Tweak for AES-XTS
128 */
129struct crypto_keys {
130	union {
131		u8 key[AES_MAX_KEY_SIZE];
132		u8 key1[AES_MAX_KEY_SIZE];
133	} u;
134	u8 iv[AES_BLOCK_SIZE];
135};
136
137/**
138 * struct auth_keys - Authentication keys
139 * @ipad: IPAD or KEY2 for AES-XTS
140 * @opad: OPAD or AUTH KEY if auth_input_type = 1
141 */
142struct auth_keys {
143	union {
144		u8 ipad[64];
145		u8 key2[64];
146	} u;
147	u8 opad[64];
148};
149
150union fc_ctx_flags {
151	__be64 f;
152	u64 fu;
153	struct {
154#if defined(__BIG_ENDIAN_BITFIELD)
155		u64 cipher_type	: 4;
156		u64 reserved_59	: 1;
157		u64 aes_keylen : 2;
158		u64 iv_source : 1;
159		u64 hash_type : 4;
160		u64 reserved_49_51 : 3;
161		u64 auth_input_type: 1;
162		u64 mac_len : 8;
163		u64 reserved_0_39 : 40;
164#else
165		u64 reserved_0_39 : 40;
166		u64 mac_len : 8;
167		u64 auth_input_type: 1;
168		u64 reserved_49_51 : 3;
169		u64 hash_type : 4;
170		u64 iv_source : 1;
171		u64 aes_keylen : 2;
172		u64 reserved_59	: 1;
173		u64 cipher_type	: 4;
174#endif
175	} w0;
176};
177/**
178 * struct flexi_crypto_context - Crypto context
179 * @cipher_type: Encryption cipher type
180 * @aes_keylen: AES key length
181 * @iv_source: Encryption IV source
182 * @hash_type: Authentication type
183 * @auth_input_type: Authentication input type
184 *   1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
185 *   0 - Authentication OPAD/IPAD
186 * @mac_len: mac length
187 * @crypto: Crypto keys
188 * @auth: Authentication keys
189 */
190struct flexi_crypto_context {
191	union fc_ctx_flags flags;
192	struct crypto_keys crypto;
193	struct auth_keys auth;
194};
195
196struct crypto_ctx_hdr {
197	struct dma_pool *pool;
198	dma_addr_t dma;
199	void *vaddr;
200};
201
202struct nitrox_crypto_ctx {
203	struct nitrox_device *ndev;
204	union {
205		u64 ctx_handle;
206		struct flexi_crypto_context *fctx;
207	} u;
208	struct crypto_ctx_hdr *chdr;
209	sereq_completion_t callback;
210};
211
212struct nitrox_kcrypt_request {
213	struct se_crypto_request creq;
214	u8 *src;
215	u8 *dst;
216	u8 *iv_out;
217};
218
219/**
220 * struct nitrox_aead_rctx - AEAD request context
221 * @nkreq: Base request context
222 * @cryptlen: Encryption/Decryption data length
223 * @assoclen: AAD length
224 * @srclen: Input buffer length
225 * @dstlen: Output buffer length
226 * @iv: IV data
227 * @ivsize: IV data length
228 * @flags: AEAD req flags
229 * @ctx_handle: Device context handle
230 * @src: Source sglist
231 * @dst: Destination sglist
232 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
233 */
234struct nitrox_aead_rctx {
235	struct nitrox_kcrypt_request nkreq;
236	unsigned int cryptlen;
237	unsigned int assoclen;
238	unsigned int srclen;
239	unsigned int dstlen;
240	u8 *iv;
241	int ivsize;
242	u32 flags;
243	u64 ctx_handle;
244	struct scatterlist *src;
245	struct scatterlist *dst;
246	u8 ctrl_arg;
247};
248
249/**
250 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
251 * @base: AEAD request context
252 * @src: Source sglist
253 * @dst: Destination sglist
254 * @assoc: AAD
255 */
256struct nitrox_rfc4106_rctx {
257	struct nitrox_aead_rctx base;
258	struct scatterlist src[3];
259	struct scatterlist dst[3];
260	u8 assoc[20];
261};
262
263/**
264 * struct pkt_instr_hdr - Packet Instruction Header
265 * @g: Gather used
266 *   When [G] is set and [GSZ] != 0, the instruction is
267 *   indirect gather instruction.
268 *   When [G] is set and [GSZ] = 0, the instruction is
269 *   direct gather instruction.
270 * @gsz: Number of pointers in the indirect gather list
271 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
272 *   and adds them to the packet after the pkt_instr_hdr but before any UDD
273 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
274 *   when [IHI] is set.
275 * @fsz: The number of front data bytes directly included in the
276 *   PCIe instruction.
277 * @tlen: The length of the input packet in bytes, include:
278 *   - 16B pkt_hdr
279 *   - Inline context bytes if any,
280 *   - UDD if any,
281 *   - packet payload bytes
282 */
283union pkt_instr_hdr {
284	__be64 bev;
285	u64 value;
286	struct {
287#if defined(__BIG_ENDIAN_BITFIELD)
288		u64 raz_48_63 : 16;
289		u64 g : 1;
290		u64 gsz	: 7;
291		u64 ihi	: 1;
292		u64 ssz	: 7;
293		u64 raz_30_31 : 2;
294		u64 fsz	: 6;
295		u64 raz_16_23 : 8;
296		u64 tlen : 16;
297#else
298		u64 tlen : 16;
299		u64 raz_16_23 : 8;
300		u64 fsz	: 6;
301		u64 raz_30_31 : 2;
302		u64 ssz	: 7;
303		u64 ihi	: 1;
304		u64 gsz	: 7;
305		u64 g : 1;
306		u64 raz_48_63 : 16;
307#endif
308	} s;
309};
310
311/**
312 * struct pkt_hdr - Packet Input Header
313 * @opcode: Request opcode (Major)
314 * @arg: Request opcode (Minor)
315 * @ctxc: Context control.
316 * @unca: When set [UNC] is the uncertainty count for an input packet.
317 *        The hardware uses uncertainty counts to predict
318 *        output buffer use and avoid deadlock.
319 * @info: Not used by input hardware. Available for use
320 *        during SE processing.
321 * @destport: The expected destination port/ring/channel for the packet.
322 * @unc: Uncertainty count for an input packet.
323 * @grp: SE group that will process the input packet.
324 * @ctxl: Context Length in 64-bit words.
325 * @uddl: User-defined data (UDD) length in bytes.
326 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
327 */
328union pkt_hdr {
329	__be64 bev[2];
330	u64 value[2];
331	struct {
332#if defined(__BIG_ENDIAN_BITFIELD)
333		u64 opcode : 8;
334		u64 arg	: 8;
335		u64 ctxc : 2;
336		u64 unca : 1;
337		u64 raz_44 : 1;
338		u64 info : 3;
339		u64 destport : 9;
340		u64 unc	: 8;
341		u64 raz_19_23 : 5;
342		u64 grp	: 3;
343		u64 raz_15 : 1;
344		u64 ctxl : 7;
345		u64 uddl : 8;
346#else
347		u64 uddl : 8;
348		u64 ctxl : 7;
349		u64 raz_15 : 1;
350		u64 grp	: 3;
351		u64 raz_19_23 : 5;
352		u64 unc	: 8;
353		u64 destport : 9;
354		u64 info : 3;
355		u64 raz_44 : 1;
356		u64 unca : 1;
357		u64 ctxc : 2;
358		u64 arg	: 8;
359		u64 opcode : 8;
360#endif
361		__be64 ctxp;
362	} s;
363};
364
365/**
366 * struct slc_store_info - Solicited Paceket Output Store Information.
367 * @ssz: The number of scatterlist pointers for the solicited output port
368 *       packet.
369 * @rptr: The result pointer for the solicited output port packet.
370 *        If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
371 *        host that is large enough to hold the entire output packet.
372 *        If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
373 *        sglist components at [RPTR] on the remote host.
374 */
375union slc_store_info {
376	__be64 bev[2];
377	u64 value[2];
378	struct {
379#if defined(__BIG_ENDIAN_BITFIELD)
380		u64 raz_39_63 : 25;
381		u64 ssz	: 7;
382		u64 raz_0_31 : 32;
383#else
384		u64 raz_0_31 : 32;
385		u64 ssz	: 7;
386		u64 raz_39_63 : 25;
387#endif
388		__be64 rptr;
389	} s;
390};
391
392/**
393 * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
394 * @dptr0 : Input pointer points to buffer in remote host.
395 * @ih: Packet Instruction Header (8 bytes)
396 * @irh: Packet Input Header (16 bytes)
397 * @slc: Solicited Packet Output Store Information (16 bytes)
398 * @fdata: Front data
399 *
400 * 64-Byte Instruction Format
401 */
402struct nps_pkt_instr {
403	__be64 dptr0;
404	union pkt_instr_hdr ih;
405	union pkt_hdr irh;
406	union slc_store_info slc;
407	u64 fdata[2];
408};
409
410/**
411 * struct aqmq_command_s - The 32 byte command for AE processing.
412 * @opcode: Request opcode
413 * @param1: Request control parameter 1
414 * @param2: Request control parameter 2
415 * @dlen: Input length
416 * @dptr: Input pointer points to buffer in remote host
417 * @rptr: Result pointer points to buffer in remote host
418 * @grp: AQM Group (0..7)
419 * @cptr: Context pointer
420 */
421struct aqmq_command_s {
422	__be16 opcode;
423	__be16 param1;
424	__be16 param2;
425	__be16 dlen;
426	__be64 dptr;
427	__be64 rptr;
428	union {
429		__be64 word3;
430#if defined(__BIG_ENDIAN_BITFIELD)
431		u64 grp : 3;
432		u64 cptr : 61;
433#else
434		u64 cptr : 61;
435		u64 grp : 3;
436#endif
437	};
438};
439
440/**
441 * struct ctx_hdr - Book keeping data about the crypto context
442 * @pool: Pool used to allocate crypto context
443 * @dma: Base DMA address of the crypto context
444 * @ctx_dma: Actual usable crypto context for NITROX
445 */
446struct ctx_hdr {
447	struct dma_pool *pool;
448	dma_addr_t dma;
449	dma_addr_t ctx_dma;
450};
451
452/*
453 * struct sglist_component - SG list component format
454 * @len0: The number of bytes at [PTR0] on the remote host.
455 * @len1: The number of bytes at [PTR1] on the remote host.
456 * @len2: The number of bytes at [PTR2] on the remote host.
457 * @len3: The number of bytes at [PTR3] on the remote host.
458 * @dma0: First pointer point to buffer in remote host.
459 * @dma1: Second pointer point to buffer in remote host.
460 * @dma2: Third pointer point to buffer in remote host.
461 * @dma3: Fourth pointer point to buffer in remote host.
462 */
463struct nitrox_sgcomp {
464	__be16 len[4];
465	__be64 dma[4];
466};
467
468/*
469 * strutct nitrox_sgtable - SG list information
470 * @sgmap_cnt: Number of buffers mapped
471 * @total_bytes: Total bytes in sglist.
472 * @sgcomp_len: Total sglist components length.
473 * @sgcomp_dma: DMA address of sglist component.
474 * @sg: crypto request buffer.
475 * @sgcomp: sglist component for NITROX.
476 */
477struct nitrox_sgtable {
478	u8 sgmap_cnt;
479	u16 total_bytes;
480	u32 sgcomp_len;
481	dma_addr_t sgcomp_dma;
482	struct scatterlist *sg;
483	struct nitrox_sgcomp *sgcomp;
484};
485
486/* Response Header Length */
487#define ORH_HLEN	8
488/* Completion bytes Length */
489#define COMP_HLEN	8
490
491struct resp_hdr {
492	u64 *orh;
493	u64 *completion;
494};
495
496typedef void (*completion_t)(void *arg, int err);
497
498/**
499 * struct nitrox_softreq - Represents the NIROX Request.
500 * @response: response list entry
501 * @backlog: Backlog list entry
502 * @ndev: Device used to submit the request
503 * @cmdq: Command queue for submission
504 * @resp: Response headers
505 * @instr: 64B instruction
506 * @in: SG table for input
507 * @out SG table for output
508 * @tstamp: Request submitted time in jiffies
509 * @callback: callback after request completion/timeout
510 * @cb_arg: callback argument
511 */
512struct nitrox_softreq {
513	struct list_head response;
514	struct list_head backlog;
515
516	u32 flags;
517	gfp_t gfp;
518	atomic_t status;
519
520	struct nitrox_device *ndev;
521	struct nitrox_cmdq *cmdq;
522
523	struct nps_pkt_instr instr;
524	struct resp_hdr resp;
525	struct nitrox_sgtable in;
526	struct nitrox_sgtable out;
527
528	unsigned long tstamp;
529
530	completion_t callback;
531	void *cb_arg;
532};
533
534static inline int flexi_aes_keylen(int keylen)
535{
536	int aes_keylen;
537
538	switch (keylen) {
539	case AES_KEYSIZE_128:
540		aes_keylen = 1;
541		break;
542	case AES_KEYSIZE_192:
543		aes_keylen = 2;
544		break;
545	case AES_KEYSIZE_256:
546		aes_keylen = 3;
547		break;
548	default:
549		aes_keylen = -EINVAL;
550		break;
551	}
552	return aes_keylen;
553}
554
555static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
556{
557	size_t size;
558
559	size = sizeof(struct scatterlist) * nents;
560	size += extralen;
561
562	return kzalloc(size, gfp);
563}
564
565/**
566 * create_single_sg - Point SG entry to the data
567 * @sg:		Destination SG list
568 * @buf:	Data
569 * @buflen:	Data length
570 *
571 * Returns next free entry in the destination SG list
572 **/
573static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
574						   void *buf, int buflen)
575{
576	sg_set_buf(sg, buf, buflen);
577	sg++;
578	return sg;
579}
580
581/**
582 * create_multi_sg - Create multiple sg entries with buflen data length from
583 *		     source sglist
584 * @to_sg:	Destination SG list
585 * @from_sg:	Source SG list
586 * @buflen:	Data length
587 *
588 * Returns next free entry in the destination SG list
589 **/
590static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
591						  struct scatterlist *from_sg,
592						  int buflen)
593{
594	struct scatterlist *sg = to_sg;
595	unsigned int sglen;
596
597	for (; buflen && from_sg; buflen -= sglen) {
598		sglen = from_sg->length;
599		if (sglen > buflen)
600			sglen = buflen;
601
602		sg_set_buf(sg, sg_virt(from_sg), sglen);
603		from_sg = sg_next(from_sg);
604		sg++;
605	}
606
607	return sg;
608}
609
610static inline void set_orh_value(u64 *orh)
611{
612	WRITE_ONCE(*orh, PENDING_SIG);
613}
614
615static inline void set_comp_value(u64 *comp)
616{
617	WRITE_ONCE(*comp, PENDING_SIG);
618}
619
620static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
621				    int nents, int ivsize)
622{
623	struct se_crypto_request *creq = &nkreq->creq;
624
625	nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
626	if (!nkreq->src)
627		return -ENOMEM;
628
629	return 0;
630}
631
632static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
633{
634	memcpy(dst, src, size);
635}
636
637static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
638{
639	return (struct scatterlist *)(iv + ivsize);
640}
641
642static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
643					  int nents, int ivsize,
644					  struct scatterlist *src, int buflen)
645{
646	char *iv = nkreq->src;
647	struct scatterlist *sg;
648	struct se_crypto_request *creq = &nkreq->creq;
649
650	creq->src = nitrox_creq_src_sg(iv, ivsize);
651	sg = creq->src;
652	sg_init_table(sg, nents);
653
654	/* Input format:
655	 * +----+----------------+
656	 * | IV | SRC sg entries |
657	 * +----+----------------+
658	 */
659
660	/* IV */
661	sg = create_single_sg(sg, iv, ivsize);
662	/* SRC entries */
663	create_multi_sg(sg, src, buflen);
664}
665
666static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
667				    int nents)
668{
669	int extralen = ORH_HLEN + COMP_HLEN;
670	struct se_crypto_request *creq = &nkreq->creq;
671
672	nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
673	if (!nkreq->dst)
674		return -ENOMEM;
675
676	return 0;
677}
678
679static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
680{
681	struct se_crypto_request *creq = &nkreq->creq;
682
683	creq->orh = (u64 *)(nkreq->dst);
684	set_orh_value(creq->orh);
685}
686
687static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
688{
689	struct se_crypto_request *creq = &nkreq->creq;
690
691	creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
692	set_comp_value(creq->comp);
693}
694
695static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
696{
697	return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
698}
699
700static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
701					  int nents, int ivsize,
702					  struct scatterlist *dst, int buflen)
703{
704	struct se_crypto_request *creq = &nkreq->creq;
705	struct scatterlist *sg;
706	char *iv = nkreq->src;
707
708	creq->dst = nitrox_creq_dst_sg(nkreq->dst);
709	sg = creq->dst;
710	sg_init_table(sg, nents);
711
712	/* Output format:
713	 * +-----+----+----------------+-----------------+
714	 * | ORH | IV | DST sg entries | COMPLETION Bytes|
715	 * +-----+----+----------------+-----------------+
716	 */
717
718	/* ORH */
719	sg = create_single_sg(sg, creq->orh, ORH_HLEN);
720	/* IV */
721	sg = create_single_sg(sg, iv, ivsize);
722	/* DST entries */
723	sg = create_multi_sg(sg, dst, buflen);
724	/* COMPLETION Bytes */
725	create_single_sg(sg, creq->comp, COMP_HLEN);
726}
727
728#endif /* __NITROX_REQ_H */
729