1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2/* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7#include "compat.h" 8#include "regs.h" 9#include "caamalg_qi2.h" 10#include "dpseci_cmd.h" 11#include "desc_constr.h" 12#include "error.h" 13#include "sg_sw_sec4.h" 14#include "sg_sw_qm2.h" 15#include "key_gen.h" 16#include "caamalg_desc.h" 17#include "caamhash_desc.h" 18#include "dpseci-debugfs.h" 19#include <linux/dma-mapping.h> 20#include <linux/fsl/mc.h> 21#include <linux/kernel.h> 22#include <soc/fsl/dpaa2-io.h> 23#include <soc/fsl/dpaa2-fd.h> 24#include <crypto/xts.h> 25#include <asm/unaligned.h> 26 27#define CAAM_CRA_PRIORITY 2000 28 29/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 30#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 31 SHA512_DIGEST_SIZE * 2) 32 33/* 34 * This is a cache of buffers, from which the users of CAAM QI driver 35 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 36 * NOTE: A more elegant solution would be to have some headroom in the frames 37 * being processed. This can be added by the dpaa2-eth driver. This would 38 * pose a problem for userspace application processing which cannot 39 * know of this limitation. So for now, this will work. 40 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 41 */ 42static struct kmem_cache *qi_cache; 43 44struct caam_alg_entry { 45 struct device *dev; 46 int class1_alg_type; 47 int class2_alg_type; 48 bool rfc3686; 49 bool geniv; 50 bool nodkp; 51}; 52 53struct caam_aead_alg { 54 struct aead_alg aead; 55 struct caam_alg_entry caam; 56 bool registered; 57}; 58 59struct caam_skcipher_alg { 60 struct skcipher_alg skcipher; 61 struct caam_alg_entry caam; 62 bool registered; 63}; 64 65/** 66 * struct caam_ctx - per-session context 67 * @flc: Flow Contexts array 68 * @key: [authentication key], encryption key 69 * @flc_dma: I/O virtual addresses of the Flow Contexts 70 * @key_dma: I/O virtual address of the key 71 * @dir: DMA direction for mapping key and Flow Contexts 72 * @dev: dpseci device 73 * @adata: authentication algorithm details 74 * @cdata: encryption algorithm details 75 * @authsize: authentication tag (a.k.a. ICV / MAC) size 76 * @xts_key_fallback: true if fallback tfm needs to be used due 77 * to unsupported xts key lengths 78 * @fallback: xts fallback tfm 79 */ 80struct caam_ctx { 81 struct caam_flc flc[NUM_OP]; 82 u8 key[CAAM_MAX_KEY_SIZE]; 83 dma_addr_t flc_dma[NUM_OP]; 84 dma_addr_t key_dma; 85 enum dma_data_direction dir; 86 struct device *dev; 87 struct alginfo adata; 88 struct alginfo cdata; 89 unsigned int authsize; 90 bool xts_key_fallback; 91 struct crypto_skcipher *fallback; 92}; 93 94static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 95 dma_addr_t iova_addr) 96{ 97 phys_addr_t phys_addr; 98 99 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 100 iova_addr; 101 102 return phys_to_virt(phys_addr); 103} 104 105/* 106 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 107 * 108 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 109 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 110 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 111 * hosting 16 SG entries. 112 * 113 * @flags - flags that would be used for the equivalent kmalloc(..) call 114 * 115 * Returns a pointer to a retrieved buffer on success or NULL on failure. 116 */ 117static inline void *qi_cache_zalloc(gfp_t flags) 118{ 119 return kmem_cache_zalloc(qi_cache, flags); 120} 121 122/* 123 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 124 * 125 * @obj - buffer previously allocated by qi_cache_zalloc 126 * 127 * No checking is being done, the call is a passthrough call to 128 * kmem_cache_free(...) 129 */ 130static inline void qi_cache_free(void *obj) 131{ 132 kmem_cache_free(qi_cache, obj); 133} 134 135static struct caam_request *to_caam_req(struct crypto_async_request *areq) 136{ 137 switch (crypto_tfm_alg_type(areq->tfm)) { 138 case CRYPTO_ALG_TYPE_SKCIPHER: 139 return skcipher_request_ctx_dma(skcipher_request_cast(areq)); 140 case CRYPTO_ALG_TYPE_AEAD: 141 return aead_request_ctx_dma( 142 container_of(areq, struct aead_request, base)); 143 case CRYPTO_ALG_TYPE_AHASH: 144 return ahash_request_ctx_dma(ahash_request_cast(areq)); 145 default: 146 return ERR_PTR(-EINVAL); 147 } 148} 149 150static void caam_unmap(struct device *dev, struct scatterlist *src, 151 struct scatterlist *dst, int src_nents, 152 int dst_nents, dma_addr_t iv_dma, int ivsize, 153 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 154 int qm_sg_bytes) 155{ 156 if (dst != src) { 157 if (src_nents) 158 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 159 if (dst_nents) 160 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 161 } else { 162 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 163 } 164 165 if (iv_dma) 166 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 167 168 if (qm_sg_bytes) 169 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 170} 171 172static int aead_set_sh_desc(struct crypto_aead *aead) 173{ 174 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 175 typeof(*alg), aead); 176 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 177 unsigned int ivsize = crypto_aead_ivsize(aead); 178 struct device *dev = ctx->dev; 179 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 180 struct caam_flc *flc; 181 u32 *desc; 182 u32 ctx1_iv_off = 0; 183 u32 *nonce = NULL; 184 unsigned int data_len[2]; 185 u32 inl_mask; 186 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 187 OP_ALG_AAI_CTR_MOD128); 188 const bool is_rfc3686 = alg->caam.rfc3686; 189 190 if (!ctx->cdata.keylen || !ctx->authsize) 191 return 0; 192 193 /* 194 * AES-CTR needs to load IV in CONTEXT1 reg 195 * at an offset of 128bits (16bytes) 196 * CONTEXT1[255:128] = IV 197 */ 198 if (ctr_mode) 199 ctx1_iv_off = 16; 200 201 /* 202 * RFC3686 specific: 203 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 204 */ 205 if (is_rfc3686) { 206 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 207 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 208 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 209 } 210 211 /* 212 * In case |user key| > |derived key|, using DKP<imm,imm> would result 213 * in invalid opcodes (last bytes of user key) in the resulting 214 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 215 * addresses are needed. 216 */ 217 ctx->adata.key_virt = ctx->key; 218 ctx->adata.key_dma = ctx->key_dma; 219 220 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 221 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 222 223 data_len[0] = ctx->adata.keylen_pad; 224 data_len[1] = ctx->cdata.keylen; 225 226 /* aead_encrypt shared descriptor */ 227 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 228 DESC_QI_AEAD_ENC_LEN) + 229 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 230 DESC_JOB_IO_LEN, data_len, &inl_mask, 231 ARRAY_SIZE(data_len)) < 0) 232 return -EINVAL; 233 234 ctx->adata.key_inline = !!(inl_mask & 1); 235 ctx->cdata.key_inline = !!(inl_mask & 2); 236 237 flc = &ctx->flc[ENCRYPT]; 238 desc = flc->sh_desc; 239 240 if (alg->caam.geniv) 241 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 242 ivsize, ctx->authsize, is_rfc3686, 243 nonce, ctx1_iv_off, true, 244 priv->sec_attr.era); 245 else 246 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 247 ivsize, ctx->authsize, is_rfc3686, nonce, 248 ctx1_iv_off, true, priv->sec_attr.era); 249 250 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 251 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 252 sizeof(flc->flc) + desc_bytes(desc), 253 ctx->dir); 254 255 /* aead_decrypt shared descriptor */ 256 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 257 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 258 DESC_JOB_IO_LEN, data_len, &inl_mask, 259 ARRAY_SIZE(data_len)) < 0) 260 return -EINVAL; 261 262 ctx->adata.key_inline = !!(inl_mask & 1); 263 ctx->cdata.key_inline = !!(inl_mask & 2); 264 265 flc = &ctx->flc[DECRYPT]; 266 desc = flc->sh_desc; 267 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 268 ivsize, ctx->authsize, alg->caam.geniv, 269 is_rfc3686, nonce, ctx1_iv_off, true, 270 priv->sec_attr.era); 271 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 272 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 273 sizeof(flc->flc) + desc_bytes(desc), 274 ctx->dir); 275 276 return 0; 277} 278 279static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 280{ 281 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 282 283 ctx->authsize = authsize; 284 aead_set_sh_desc(authenc); 285 286 return 0; 287} 288 289static int aead_setkey(struct crypto_aead *aead, const u8 *key, 290 unsigned int keylen) 291{ 292 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 293 struct device *dev = ctx->dev; 294 struct crypto_authenc_keys keys; 295 296 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 297 goto badkey; 298 299 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 300 keys.authkeylen + keys.enckeylen, keys.enckeylen, 301 keys.authkeylen); 302 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 303 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 304 305 ctx->adata.keylen = keys.authkeylen; 306 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 307 OP_ALG_ALGSEL_MASK); 308 309 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 310 goto badkey; 311 312 memcpy(ctx->key, keys.authkey, keys.authkeylen); 313 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 314 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 315 keys.enckeylen, ctx->dir); 316 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 317 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 318 ctx->adata.keylen_pad + keys.enckeylen, 1); 319 320 ctx->cdata.keylen = keys.enckeylen; 321 322 memzero_explicit(&keys, sizeof(keys)); 323 return aead_set_sh_desc(aead); 324badkey: 325 memzero_explicit(&keys, sizeof(keys)); 326 return -EINVAL; 327} 328 329static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 330 unsigned int keylen) 331{ 332 struct crypto_authenc_keys keys; 333 int err; 334 335 err = crypto_authenc_extractkeys(&keys, key, keylen); 336 if (unlikely(err)) 337 goto out; 338 339 err = -EINVAL; 340 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 341 goto out; 342 343 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 344 aead_setkey(aead, key, keylen); 345 346out: 347 memzero_explicit(&keys, sizeof(keys)); 348 return err; 349} 350 351static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 352 bool encrypt) 353{ 354 struct crypto_aead *aead = crypto_aead_reqtfm(req); 355 struct caam_request *req_ctx = aead_request_ctx_dma(req); 356 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 357 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 358 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 359 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 360 typeof(*alg), aead); 361 struct device *dev = ctx->dev; 362 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 363 GFP_KERNEL : GFP_ATOMIC; 364 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 365 int src_len, dst_len = 0; 366 struct aead_edesc *edesc; 367 dma_addr_t qm_sg_dma, iv_dma = 0; 368 int ivsize = 0; 369 unsigned int authsize = ctx->authsize; 370 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 371 int in_len, out_len; 372 struct dpaa2_sg_entry *sg_table; 373 374 /* allocate space for base edesc, link tables and IV */ 375 edesc = qi_cache_zalloc(flags); 376 if (unlikely(!edesc)) { 377 dev_err(dev, "could not allocate extended descriptor\n"); 378 return ERR_PTR(-ENOMEM); 379 } 380 381 if (unlikely(req->dst != req->src)) { 382 src_len = req->assoclen + req->cryptlen; 383 dst_len = src_len + (encrypt ? authsize : (-authsize)); 384 385 src_nents = sg_nents_for_len(req->src, src_len); 386 if (unlikely(src_nents < 0)) { 387 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 388 src_len); 389 qi_cache_free(edesc); 390 return ERR_PTR(src_nents); 391 } 392 393 dst_nents = sg_nents_for_len(req->dst, dst_len); 394 if (unlikely(dst_nents < 0)) { 395 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 396 dst_len); 397 qi_cache_free(edesc); 398 return ERR_PTR(dst_nents); 399 } 400 401 if (src_nents) { 402 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 403 DMA_TO_DEVICE); 404 if (unlikely(!mapped_src_nents)) { 405 dev_err(dev, "unable to map source\n"); 406 qi_cache_free(edesc); 407 return ERR_PTR(-ENOMEM); 408 } 409 } else { 410 mapped_src_nents = 0; 411 } 412 413 if (dst_nents) { 414 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 415 DMA_FROM_DEVICE); 416 if (unlikely(!mapped_dst_nents)) { 417 dev_err(dev, "unable to map destination\n"); 418 dma_unmap_sg(dev, req->src, src_nents, 419 DMA_TO_DEVICE); 420 qi_cache_free(edesc); 421 return ERR_PTR(-ENOMEM); 422 } 423 } else { 424 mapped_dst_nents = 0; 425 } 426 } else { 427 src_len = req->assoclen + req->cryptlen + 428 (encrypt ? authsize : 0); 429 430 src_nents = sg_nents_for_len(req->src, src_len); 431 if (unlikely(src_nents < 0)) { 432 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 433 src_len); 434 qi_cache_free(edesc); 435 return ERR_PTR(src_nents); 436 } 437 438 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 439 DMA_BIDIRECTIONAL); 440 if (unlikely(!mapped_src_nents)) { 441 dev_err(dev, "unable to map source\n"); 442 qi_cache_free(edesc); 443 return ERR_PTR(-ENOMEM); 444 } 445 } 446 447 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 448 ivsize = crypto_aead_ivsize(aead); 449 450 /* 451 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 452 * Input is not contiguous. 453 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 454 * the end of the table by allocating more S/G entries. Logic: 455 * if (src != dst && output S/G) 456 * pad output S/G, if needed 457 * else if (src == dst && S/G) 458 * overlapping S/Gs; pad one of them 459 * else if (input S/G) ... 460 * pad input S/G, if needed 461 */ 462 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 463 if (mapped_dst_nents > 1) 464 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 465 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 466 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 467 1 + !!ivsize + 468 pad_sg_nents(mapped_src_nents)); 469 else 470 qm_sg_nents = pad_sg_nents(qm_sg_nents); 471 472 sg_table = &edesc->sgt[0]; 473 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 474 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 475 CAAM_QI_MEMCACHE_SIZE)) { 476 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 477 qm_sg_nents, ivsize); 478 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 479 0, DMA_NONE, 0, 0); 480 qi_cache_free(edesc); 481 return ERR_PTR(-ENOMEM); 482 } 483 484 if (ivsize) { 485 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 486 487 /* Make sure IV is located in a DMAable area */ 488 memcpy(iv, req->iv, ivsize); 489 490 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 491 if (dma_mapping_error(dev, iv_dma)) { 492 dev_err(dev, "unable to map IV\n"); 493 caam_unmap(dev, req->src, req->dst, src_nents, 494 dst_nents, 0, 0, DMA_NONE, 0, 0); 495 qi_cache_free(edesc); 496 return ERR_PTR(-ENOMEM); 497 } 498 } 499 500 edesc->src_nents = src_nents; 501 edesc->dst_nents = dst_nents; 502 edesc->iv_dma = iv_dma; 503 504 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 505 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 506 /* 507 * The associated data comes already with the IV but we need 508 * to skip it when we authenticate or encrypt... 509 */ 510 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 511 else 512 edesc->assoclen = cpu_to_caam32(req->assoclen); 513 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 514 DMA_TO_DEVICE); 515 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 516 dev_err(dev, "unable to map assoclen\n"); 517 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 518 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 519 qi_cache_free(edesc); 520 return ERR_PTR(-ENOMEM); 521 } 522 523 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 524 qm_sg_index++; 525 if (ivsize) { 526 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 527 qm_sg_index++; 528 } 529 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 530 qm_sg_index += mapped_src_nents; 531 532 if (mapped_dst_nents > 1) 533 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 534 535 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 536 if (dma_mapping_error(dev, qm_sg_dma)) { 537 dev_err(dev, "unable to map S/G table\n"); 538 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 539 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 540 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 541 qi_cache_free(edesc); 542 return ERR_PTR(-ENOMEM); 543 } 544 545 edesc->qm_sg_dma = qm_sg_dma; 546 edesc->qm_sg_bytes = qm_sg_bytes; 547 548 out_len = req->assoclen + req->cryptlen + 549 (encrypt ? ctx->authsize : (-ctx->authsize)); 550 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 551 552 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 553 dpaa2_fl_set_final(in_fle, true); 554 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 555 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 556 dpaa2_fl_set_len(in_fle, in_len); 557 558 if (req->dst == req->src) { 559 if (mapped_src_nents == 1) { 560 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 561 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 562 } else { 563 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 564 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 565 (1 + !!ivsize) * sizeof(*sg_table)); 566 } 567 } else if (!mapped_dst_nents) { 568 /* 569 * crypto engine requires the output entry to be present when 570 * "frame list" FD is used. 571 * Since engine does not support FMT=2'b11 (unused entry type), 572 * leaving out_fle zeroized is the best option. 573 */ 574 goto skip_out_fle; 575 } else if (mapped_dst_nents == 1) { 576 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 577 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 578 } else { 579 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 580 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 581 sizeof(*sg_table)); 582 } 583 584 dpaa2_fl_set_len(out_fle, out_len); 585 586skip_out_fle: 587 return edesc; 588} 589 590static int chachapoly_set_sh_desc(struct crypto_aead *aead) 591{ 592 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 593 unsigned int ivsize = crypto_aead_ivsize(aead); 594 struct device *dev = ctx->dev; 595 struct caam_flc *flc; 596 u32 *desc; 597 598 if (!ctx->cdata.keylen || !ctx->authsize) 599 return 0; 600 601 flc = &ctx->flc[ENCRYPT]; 602 desc = flc->sh_desc; 603 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 604 ctx->authsize, true, true); 605 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 606 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 607 sizeof(flc->flc) + desc_bytes(desc), 608 ctx->dir); 609 610 flc = &ctx->flc[DECRYPT]; 611 desc = flc->sh_desc; 612 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 613 ctx->authsize, false, true); 614 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 615 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 616 sizeof(flc->flc) + desc_bytes(desc), 617 ctx->dir); 618 619 return 0; 620} 621 622static int chachapoly_setauthsize(struct crypto_aead *aead, 623 unsigned int authsize) 624{ 625 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 626 627 if (authsize != POLY1305_DIGEST_SIZE) 628 return -EINVAL; 629 630 ctx->authsize = authsize; 631 return chachapoly_set_sh_desc(aead); 632} 633 634static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 635 unsigned int keylen) 636{ 637 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 638 unsigned int ivsize = crypto_aead_ivsize(aead); 639 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 640 641 if (keylen != CHACHA_KEY_SIZE + saltlen) 642 return -EINVAL; 643 644 memcpy(ctx->key, key, keylen); 645 ctx->cdata.key_virt = ctx->key; 646 ctx->cdata.keylen = keylen - saltlen; 647 648 return chachapoly_set_sh_desc(aead); 649} 650 651static int gcm_set_sh_desc(struct crypto_aead *aead) 652{ 653 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 654 struct device *dev = ctx->dev; 655 unsigned int ivsize = crypto_aead_ivsize(aead); 656 struct caam_flc *flc; 657 u32 *desc; 658 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 659 ctx->cdata.keylen; 660 661 if (!ctx->cdata.keylen || !ctx->authsize) 662 return 0; 663 664 /* 665 * AES GCM encrypt shared descriptor 666 * Job Descriptor and Shared Descriptor 667 * must fit into the 64-word Descriptor h/w Buffer 668 */ 669 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 670 ctx->cdata.key_inline = true; 671 ctx->cdata.key_virt = ctx->key; 672 } else { 673 ctx->cdata.key_inline = false; 674 ctx->cdata.key_dma = ctx->key_dma; 675 } 676 677 flc = &ctx->flc[ENCRYPT]; 678 desc = flc->sh_desc; 679 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 680 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 681 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 682 sizeof(flc->flc) + desc_bytes(desc), 683 ctx->dir); 684 685 /* 686 * Job Descriptor and Shared Descriptors 687 * must all fit into the 64-word Descriptor h/w Buffer 688 */ 689 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 690 ctx->cdata.key_inline = true; 691 ctx->cdata.key_virt = ctx->key; 692 } else { 693 ctx->cdata.key_inline = false; 694 ctx->cdata.key_dma = ctx->key_dma; 695 } 696 697 flc = &ctx->flc[DECRYPT]; 698 desc = flc->sh_desc; 699 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 700 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 701 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 702 sizeof(flc->flc) + desc_bytes(desc), 703 ctx->dir); 704 705 return 0; 706} 707 708static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 709{ 710 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 711 int err; 712 713 err = crypto_gcm_check_authsize(authsize); 714 if (err) 715 return err; 716 717 ctx->authsize = authsize; 718 gcm_set_sh_desc(authenc); 719 720 return 0; 721} 722 723static int gcm_setkey(struct crypto_aead *aead, 724 const u8 *key, unsigned int keylen) 725{ 726 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 727 struct device *dev = ctx->dev; 728 int ret; 729 730 ret = aes_check_keylen(keylen); 731 if (ret) 732 return ret; 733 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 734 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 735 736 memcpy(ctx->key, key, keylen); 737 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 738 ctx->cdata.keylen = keylen; 739 740 return gcm_set_sh_desc(aead); 741} 742 743static int rfc4106_set_sh_desc(struct crypto_aead *aead) 744{ 745 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 746 struct device *dev = ctx->dev; 747 unsigned int ivsize = crypto_aead_ivsize(aead); 748 struct caam_flc *flc; 749 u32 *desc; 750 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 751 ctx->cdata.keylen; 752 753 if (!ctx->cdata.keylen || !ctx->authsize) 754 return 0; 755 756 ctx->cdata.key_virt = ctx->key; 757 758 /* 759 * RFC4106 encrypt shared descriptor 760 * Job Descriptor and Shared Descriptor 761 * must fit into the 64-word Descriptor h/w Buffer 762 */ 763 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 764 ctx->cdata.key_inline = true; 765 } else { 766 ctx->cdata.key_inline = false; 767 ctx->cdata.key_dma = ctx->key_dma; 768 } 769 770 flc = &ctx->flc[ENCRYPT]; 771 desc = flc->sh_desc; 772 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 773 true); 774 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 775 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 776 sizeof(flc->flc) + desc_bytes(desc), 777 ctx->dir); 778 779 /* 780 * Job Descriptor and Shared Descriptors 781 * must all fit into the 64-word Descriptor h/w Buffer 782 */ 783 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 784 ctx->cdata.key_inline = true; 785 } else { 786 ctx->cdata.key_inline = false; 787 ctx->cdata.key_dma = ctx->key_dma; 788 } 789 790 flc = &ctx->flc[DECRYPT]; 791 desc = flc->sh_desc; 792 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 793 true); 794 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 795 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 796 sizeof(flc->flc) + desc_bytes(desc), 797 ctx->dir); 798 799 return 0; 800} 801 802static int rfc4106_setauthsize(struct crypto_aead *authenc, 803 unsigned int authsize) 804{ 805 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 806 int err; 807 808 err = crypto_rfc4106_check_authsize(authsize); 809 if (err) 810 return err; 811 812 ctx->authsize = authsize; 813 rfc4106_set_sh_desc(authenc); 814 815 return 0; 816} 817 818static int rfc4106_setkey(struct crypto_aead *aead, 819 const u8 *key, unsigned int keylen) 820{ 821 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 822 struct device *dev = ctx->dev; 823 int ret; 824 825 ret = aes_check_keylen(keylen - 4); 826 if (ret) 827 return ret; 828 829 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 830 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 831 832 memcpy(ctx->key, key, keylen); 833 /* 834 * The last four bytes of the key material are used as the salt value 835 * in the nonce. Update the AES key length. 836 */ 837 ctx->cdata.keylen = keylen - 4; 838 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 839 ctx->dir); 840 841 return rfc4106_set_sh_desc(aead); 842} 843 844static int rfc4543_set_sh_desc(struct crypto_aead *aead) 845{ 846 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 847 struct device *dev = ctx->dev; 848 unsigned int ivsize = crypto_aead_ivsize(aead); 849 struct caam_flc *flc; 850 u32 *desc; 851 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 852 ctx->cdata.keylen; 853 854 if (!ctx->cdata.keylen || !ctx->authsize) 855 return 0; 856 857 ctx->cdata.key_virt = ctx->key; 858 859 /* 860 * RFC4543 encrypt shared descriptor 861 * Job Descriptor and Shared Descriptor 862 * must fit into the 64-word Descriptor h/w Buffer 863 */ 864 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 865 ctx->cdata.key_inline = true; 866 } else { 867 ctx->cdata.key_inline = false; 868 ctx->cdata.key_dma = ctx->key_dma; 869 } 870 871 flc = &ctx->flc[ENCRYPT]; 872 desc = flc->sh_desc; 873 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 874 true); 875 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 876 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 877 sizeof(flc->flc) + desc_bytes(desc), 878 ctx->dir); 879 880 /* 881 * Job Descriptor and Shared Descriptors 882 * must all fit into the 64-word Descriptor h/w Buffer 883 */ 884 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 885 ctx->cdata.key_inline = true; 886 } else { 887 ctx->cdata.key_inline = false; 888 ctx->cdata.key_dma = ctx->key_dma; 889 } 890 891 flc = &ctx->flc[DECRYPT]; 892 desc = flc->sh_desc; 893 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 894 true); 895 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 896 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 897 sizeof(flc->flc) + desc_bytes(desc), 898 ctx->dir); 899 900 return 0; 901} 902 903static int rfc4543_setauthsize(struct crypto_aead *authenc, 904 unsigned int authsize) 905{ 906 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 907 908 if (authsize != 16) 909 return -EINVAL; 910 911 ctx->authsize = authsize; 912 rfc4543_set_sh_desc(authenc); 913 914 return 0; 915} 916 917static int rfc4543_setkey(struct crypto_aead *aead, 918 const u8 *key, unsigned int keylen) 919{ 920 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 921 struct device *dev = ctx->dev; 922 int ret; 923 924 ret = aes_check_keylen(keylen - 4); 925 if (ret) 926 return ret; 927 928 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 929 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 930 931 memcpy(ctx->key, key, keylen); 932 /* 933 * The last four bytes of the key material are used as the salt value 934 * in the nonce. Update the AES key length. 935 */ 936 ctx->cdata.keylen = keylen - 4; 937 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 938 ctx->dir); 939 940 return rfc4543_set_sh_desc(aead); 941} 942 943static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 944 unsigned int keylen, const u32 ctx1_iv_off) 945{ 946 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 947 struct caam_skcipher_alg *alg = 948 container_of(crypto_skcipher_alg(skcipher), 949 struct caam_skcipher_alg, skcipher); 950 struct device *dev = ctx->dev; 951 struct caam_flc *flc; 952 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 953 u32 *desc; 954 const bool is_rfc3686 = alg->caam.rfc3686; 955 956 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 957 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 958 959 ctx->cdata.keylen = keylen; 960 ctx->cdata.key_virt = key; 961 ctx->cdata.key_inline = true; 962 963 /* skcipher_encrypt shared descriptor */ 964 flc = &ctx->flc[ENCRYPT]; 965 desc = flc->sh_desc; 966 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 967 ctx1_iv_off); 968 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 969 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 970 sizeof(flc->flc) + desc_bytes(desc), 971 ctx->dir); 972 973 /* skcipher_decrypt shared descriptor */ 974 flc = &ctx->flc[DECRYPT]; 975 desc = flc->sh_desc; 976 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 977 ctx1_iv_off); 978 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 979 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 980 sizeof(flc->flc) + desc_bytes(desc), 981 ctx->dir); 982 983 return 0; 984} 985 986static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 987 const u8 *key, unsigned int keylen) 988{ 989 int err; 990 991 err = aes_check_keylen(keylen); 992 if (err) 993 return err; 994 995 return skcipher_setkey(skcipher, key, keylen, 0); 996} 997 998static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 999 const u8 *key, unsigned int keylen) 1000{ 1001 u32 ctx1_iv_off; 1002 int err; 1003 1004 /* 1005 * RFC3686 specific: 1006 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1007 * | *key = {KEY, NONCE} 1008 */ 1009 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1010 keylen -= CTR_RFC3686_NONCE_SIZE; 1011 1012 err = aes_check_keylen(keylen); 1013 if (err) 1014 return err; 1015 1016 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1017} 1018 1019static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1020 const u8 *key, unsigned int keylen) 1021{ 1022 u32 ctx1_iv_off; 1023 int err; 1024 1025 /* 1026 * AES-CTR needs to load IV in CONTEXT1 reg 1027 * at an offset of 128bits (16bytes) 1028 * CONTEXT1[255:128] = IV 1029 */ 1030 ctx1_iv_off = 16; 1031 1032 err = aes_check_keylen(keylen); 1033 if (err) 1034 return err; 1035 1036 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1037} 1038 1039static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1040 const u8 *key, unsigned int keylen) 1041{ 1042 if (keylen != CHACHA_KEY_SIZE) 1043 return -EINVAL; 1044 1045 return skcipher_setkey(skcipher, key, keylen, 0); 1046} 1047 1048static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1049 const u8 *key, unsigned int keylen) 1050{ 1051 return verify_skcipher_des_key(skcipher, key) ?: 1052 skcipher_setkey(skcipher, key, keylen, 0); 1053} 1054 1055static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1056 const u8 *key, unsigned int keylen) 1057{ 1058 return verify_skcipher_des3_key(skcipher, key) ?: 1059 skcipher_setkey(skcipher, key, keylen, 0); 1060} 1061 1062static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1063 unsigned int keylen) 1064{ 1065 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1066 struct device *dev = ctx->dev; 1067 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 1068 struct caam_flc *flc; 1069 u32 *desc; 1070 int err; 1071 1072 err = xts_verify_key(skcipher, key, keylen); 1073 if (err) { 1074 dev_dbg(dev, "key size mismatch\n"); 1075 return err; 1076 } 1077 1078 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 1079 ctx->xts_key_fallback = true; 1080 1081 if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { 1082 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 1083 if (err) 1084 return err; 1085 } 1086 1087 ctx->cdata.keylen = keylen; 1088 ctx->cdata.key_virt = key; 1089 ctx->cdata.key_inline = true; 1090 1091 /* xts_skcipher_encrypt shared descriptor */ 1092 flc = &ctx->flc[ENCRYPT]; 1093 desc = flc->sh_desc; 1094 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1095 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1096 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1097 sizeof(flc->flc) + desc_bytes(desc), 1098 ctx->dir); 1099 1100 /* xts_skcipher_decrypt shared descriptor */ 1101 flc = &ctx->flc[DECRYPT]; 1102 desc = flc->sh_desc; 1103 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1104 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1105 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1106 sizeof(flc->flc) + desc_bytes(desc), 1107 ctx->dir); 1108 1109 return 0; 1110} 1111 1112static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1113{ 1114 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1115 struct caam_request *req_ctx = skcipher_request_ctx_dma(req); 1116 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1117 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1118 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1119 struct device *dev = ctx->dev; 1120 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1121 GFP_KERNEL : GFP_ATOMIC; 1122 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1123 struct skcipher_edesc *edesc; 1124 dma_addr_t iv_dma; 1125 u8 *iv; 1126 int ivsize = crypto_skcipher_ivsize(skcipher); 1127 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1128 struct dpaa2_sg_entry *sg_table; 1129 1130 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1131 if (unlikely(src_nents < 0)) { 1132 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1133 req->cryptlen); 1134 return ERR_PTR(src_nents); 1135 } 1136 1137 if (unlikely(req->dst != req->src)) { 1138 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1139 if (unlikely(dst_nents < 0)) { 1140 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1141 req->cryptlen); 1142 return ERR_PTR(dst_nents); 1143 } 1144 1145 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1146 DMA_TO_DEVICE); 1147 if (unlikely(!mapped_src_nents)) { 1148 dev_err(dev, "unable to map source\n"); 1149 return ERR_PTR(-ENOMEM); 1150 } 1151 1152 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1153 DMA_FROM_DEVICE); 1154 if (unlikely(!mapped_dst_nents)) { 1155 dev_err(dev, "unable to map destination\n"); 1156 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1157 return ERR_PTR(-ENOMEM); 1158 } 1159 } else { 1160 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1161 DMA_BIDIRECTIONAL); 1162 if (unlikely(!mapped_src_nents)) { 1163 dev_err(dev, "unable to map source\n"); 1164 return ERR_PTR(-ENOMEM); 1165 } 1166 } 1167 1168 qm_sg_ents = 1 + mapped_src_nents; 1169 dst_sg_idx = qm_sg_ents; 1170 1171 /* 1172 * Input, output HW S/G tables: [IV, src][dst, IV] 1173 * IV entries point to the same buffer 1174 * If src == dst, S/G entries are reused (S/G tables overlap) 1175 * 1176 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1177 * the end of the table by allocating more S/G entries. 1178 */ 1179 if (req->src != req->dst) 1180 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1181 else 1182 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1183 1184 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1185 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1186 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1187 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1188 qm_sg_ents, ivsize); 1189 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1190 0, DMA_NONE, 0, 0); 1191 return ERR_PTR(-ENOMEM); 1192 } 1193 1194 /* allocate space for base edesc, link tables and IV */ 1195 edesc = qi_cache_zalloc(flags); 1196 if (unlikely(!edesc)) { 1197 dev_err(dev, "could not allocate extended descriptor\n"); 1198 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1199 0, DMA_NONE, 0, 0); 1200 return ERR_PTR(-ENOMEM); 1201 } 1202 1203 /* Make sure IV is located in a DMAable area */ 1204 sg_table = &edesc->sgt[0]; 1205 iv = (u8 *)(sg_table + qm_sg_ents); 1206 memcpy(iv, req->iv, ivsize); 1207 1208 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1209 if (dma_mapping_error(dev, iv_dma)) { 1210 dev_err(dev, "unable to map IV\n"); 1211 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1212 0, DMA_NONE, 0, 0); 1213 qi_cache_free(edesc); 1214 return ERR_PTR(-ENOMEM); 1215 } 1216 1217 edesc->src_nents = src_nents; 1218 edesc->dst_nents = dst_nents; 1219 edesc->iv_dma = iv_dma; 1220 edesc->qm_sg_bytes = qm_sg_bytes; 1221 1222 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1223 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1224 1225 if (req->src != req->dst) 1226 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1227 1228 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1229 ivsize, 0); 1230 1231 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1232 DMA_TO_DEVICE); 1233 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1234 dev_err(dev, "unable to map S/G table\n"); 1235 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1236 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1237 qi_cache_free(edesc); 1238 return ERR_PTR(-ENOMEM); 1239 } 1240 1241 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1242 dpaa2_fl_set_final(in_fle, true); 1243 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1244 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1245 1246 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1247 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1248 1249 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1250 1251 if (req->src == req->dst) 1252 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1253 sizeof(*sg_table)); 1254 else 1255 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1256 sizeof(*sg_table)); 1257 1258 return edesc; 1259} 1260 1261static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1262 struct aead_request *req) 1263{ 1264 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1265 int ivsize = crypto_aead_ivsize(aead); 1266 1267 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1268 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1269 edesc->qm_sg_bytes); 1270 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1271} 1272 1273static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1274 struct skcipher_request *req) 1275{ 1276 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1277 int ivsize = crypto_skcipher_ivsize(skcipher); 1278 1279 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1280 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1281 edesc->qm_sg_bytes); 1282} 1283 1284static void aead_encrypt_done(void *cbk_ctx, u32 status) 1285{ 1286 struct crypto_async_request *areq = cbk_ctx; 1287 struct aead_request *req = container_of(areq, struct aead_request, 1288 base); 1289 struct caam_request *req_ctx = to_caam_req(areq); 1290 struct aead_edesc *edesc = req_ctx->edesc; 1291 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1292 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1293 int ecode = 0; 1294 1295 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1296 1297 if (unlikely(status)) 1298 ecode = caam_qi2_strstatus(ctx->dev, status); 1299 1300 aead_unmap(ctx->dev, edesc, req); 1301 qi_cache_free(edesc); 1302 aead_request_complete(req, ecode); 1303} 1304 1305static void aead_decrypt_done(void *cbk_ctx, u32 status) 1306{ 1307 struct crypto_async_request *areq = cbk_ctx; 1308 struct aead_request *req = container_of(areq, struct aead_request, 1309 base); 1310 struct caam_request *req_ctx = to_caam_req(areq); 1311 struct aead_edesc *edesc = req_ctx->edesc; 1312 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1313 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1314 int ecode = 0; 1315 1316 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1317 1318 if (unlikely(status)) 1319 ecode = caam_qi2_strstatus(ctx->dev, status); 1320 1321 aead_unmap(ctx->dev, edesc, req); 1322 qi_cache_free(edesc); 1323 aead_request_complete(req, ecode); 1324} 1325 1326static int aead_encrypt(struct aead_request *req) 1327{ 1328 struct aead_edesc *edesc; 1329 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1330 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1331 struct caam_request *caam_req = aead_request_ctx_dma(req); 1332 int ret; 1333 1334 /* allocate extended descriptor */ 1335 edesc = aead_edesc_alloc(req, true); 1336 if (IS_ERR(edesc)) 1337 return PTR_ERR(edesc); 1338 1339 caam_req->flc = &ctx->flc[ENCRYPT]; 1340 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1341 caam_req->cbk = aead_encrypt_done; 1342 caam_req->ctx = &req->base; 1343 caam_req->edesc = edesc; 1344 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1345 if (ret != -EINPROGRESS && 1346 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1347 aead_unmap(ctx->dev, edesc, req); 1348 qi_cache_free(edesc); 1349 } 1350 1351 return ret; 1352} 1353 1354static int aead_decrypt(struct aead_request *req) 1355{ 1356 struct aead_edesc *edesc; 1357 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1358 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1359 struct caam_request *caam_req = aead_request_ctx_dma(req); 1360 int ret; 1361 1362 /* allocate extended descriptor */ 1363 edesc = aead_edesc_alloc(req, false); 1364 if (IS_ERR(edesc)) 1365 return PTR_ERR(edesc); 1366 1367 caam_req->flc = &ctx->flc[DECRYPT]; 1368 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1369 caam_req->cbk = aead_decrypt_done; 1370 caam_req->ctx = &req->base; 1371 caam_req->edesc = edesc; 1372 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1373 if (ret != -EINPROGRESS && 1374 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1375 aead_unmap(ctx->dev, edesc, req); 1376 qi_cache_free(edesc); 1377 } 1378 1379 return ret; 1380} 1381 1382static int ipsec_gcm_encrypt(struct aead_request *req) 1383{ 1384 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1385} 1386 1387static int ipsec_gcm_decrypt(struct aead_request *req) 1388{ 1389 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1390} 1391 1392static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1393{ 1394 struct crypto_async_request *areq = cbk_ctx; 1395 struct skcipher_request *req = skcipher_request_cast(areq); 1396 struct caam_request *req_ctx = to_caam_req(areq); 1397 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1398 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1399 struct skcipher_edesc *edesc = req_ctx->edesc; 1400 int ecode = 0; 1401 int ivsize = crypto_skcipher_ivsize(skcipher); 1402 1403 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1404 1405 if (unlikely(status)) 1406 ecode = caam_qi2_strstatus(ctx->dev, status); 1407 1408 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1409 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1410 edesc->src_nents > 1 ? 100 : ivsize, 1); 1411 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1412 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1413 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1414 1415 skcipher_unmap(ctx->dev, edesc, req); 1416 1417 /* 1418 * The crypto API expects us to set the IV (req->iv) to the last 1419 * ciphertext block (CBC mode) or last counter (CTR mode). 1420 * This is used e.g. by the CTS mode. 1421 */ 1422 if (!ecode) 1423 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1424 ivsize); 1425 1426 qi_cache_free(edesc); 1427 skcipher_request_complete(req, ecode); 1428} 1429 1430static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1431{ 1432 struct crypto_async_request *areq = cbk_ctx; 1433 struct skcipher_request *req = skcipher_request_cast(areq); 1434 struct caam_request *req_ctx = to_caam_req(areq); 1435 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1436 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1437 struct skcipher_edesc *edesc = req_ctx->edesc; 1438 int ecode = 0; 1439 int ivsize = crypto_skcipher_ivsize(skcipher); 1440 1441 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1442 1443 if (unlikely(status)) 1444 ecode = caam_qi2_strstatus(ctx->dev, status); 1445 1446 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1447 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1448 edesc->src_nents > 1 ? 100 : ivsize, 1); 1449 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1450 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1451 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1452 1453 skcipher_unmap(ctx->dev, edesc, req); 1454 1455 /* 1456 * The crypto API expects us to set the IV (req->iv) to the last 1457 * ciphertext block (CBC mode) or last counter (CTR mode). 1458 * This is used e.g. by the CTS mode. 1459 */ 1460 if (!ecode) 1461 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1462 ivsize); 1463 1464 qi_cache_free(edesc); 1465 skcipher_request_complete(req, ecode); 1466} 1467 1468static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1469{ 1470 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1471 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1472 1473 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1474} 1475 1476static int skcipher_encrypt(struct skcipher_request *req) 1477{ 1478 struct skcipher_edesc *edesc; 1479 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1480 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1481 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1482 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1483 int ret; 1484 1485 /* 1486 * XTS is expected to return an error even for input length = 0 1487 * Note that the case input length < block size will be caught during 1488 * HW offloading and return an error. 1489 */ 1490 if (!req->cryptlen && !ctx->fallback) 1491 return 0; 1492 1493 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1494 ctx->xts_key_fallback)) { 1495 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1496 skcipher_request_set_callback(&caam_req->fallback_req, 1497 req->base.flags, 1498 req->base.complete, 1499 req->base.data); 1500 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1501 req->dst, req->cryptlen, req->iv); 1502 1503 return crypto_skcipher_encrypt(&caam_req->fallback_req); 1504 } 1505 1506 /* allocate extended descriptor */ 1507 edesc = skcipher_edesc_alloc(req); 1508 if (IS_ERR(edesc)) 1509 return PTR_ERR(edesc); 1510 1511 caam_req->flc = &ctx->flc[ENCRYPT]; 1512 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1513 caam_req->cbk = skcipher_encrypt_done; 1514 caam_req->ctx = &req->base; 1515 caam_req->edesc = edesc; 1516 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1517 if (ret != -EINPROGRESS && 1518 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1519 skcipher_unmap(ctx->dev, edesc, req); 1520 qi_cache_free(edesc); 1521 } 1522 1523 return ret; 1524} 1525 1526static int skcipher_decrypt(struct skcipher_request *req) 1527{ 1528 struct skcipher_edesc *edesc; 1529 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1530 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1531 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1532 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1533 int ret; 1534 1535 /* 1536 * XTS is expected to return an error even for input length = 0 1537 * Note that the case input length < block size will be caught during 1538 * HW offloading and return an error. 1539 */ 1540 if (!req->cryptlen && !ctx->fallback) 1541 return 0; 1542 1543 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1544 ctx->xts_key_fallback)) { 1545 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1546 skcipher_request_set_callback(&caam_req->fallback_req, 1547 req->base.flags, 1548 req->base.complete, 1549 req->base.data); 1550 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1551 req->dst, req->cryptlen, req->iv); 1552 1553 return crypto_skcipher_decrypt(&caam_req->fallback_req); 1554 } 1555 1556 /* allocate extended descriptor */ 1557 edesc = skcipher_edesc_alloc(req); 1558 if (IS_ERR(edesc)) 1559 return PTR_ERR(edesc); 1560 1561 caam_req->flc = &ctx->flc[DECRYPT]; 1562 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1563 caam_req->cbk = skcipher_decrypt_done; 1564 caam_req->ctx = &req->base; 1565 caam_req->edesc = edesc; 1566 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1567 if (ret != -EINPROGRESS && 1568 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1569 skcipher_unmap(ctx->dev, edesc, req); 1570 qi_cache_free(edesc); 1571 } 1572 1573 return ret; 1574} 1575 1576static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1577 bool uses_dkp) 1578{ 1579 dma_addr_t dma_addr; 1580 int i; 1581 1582 /* copy descriptor header template value */ 1583 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1584 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1585 1586 ctx->dev = caam->dev; 1587 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1588 1589 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1590 offsetof(struct caam_ctx, flc_dma), 1591 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1592 if (dma_mapping_error(ctx->dev, dma_addr)) { 1593 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1594 return -ENOMEM; 1595 } 1596 1597 for (i = 0; i < NUM_OP; i++) 1598 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1599 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1600 1601 return 0; 1602} 1603 1604static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1605{ 1606 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1607 struct caam_skcipher_alg *caam_alg = 1608 container_of(alg, typeof(*caam_alg), skcipher); 1609 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1610 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 1611 int ret = 0; 1612 1613 if (alg_aai == OP_ALG_AAI_XTS) { 1614 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1615 struct crypto_skcipher *fallback; 1616 1617 fallback = crypto_alloc_skcipher(tfm_name, 0, 1618 CRYPTO_ALG_NEED_FALLBACK); 1619 if (IS_ERR(fallback)) { 1620 dev_err(caam_alg->caam.dev, 1621 "Failed to allocate %s fallback: %ld\n", 1622 tfm_name, PTR_ERR(fallback)); 1623 return PTR_ERR(fallback); 1624 } 1625 1626 ctx->fallback = fallback; 1627 crypto_skcipher_set_reqsize_dma( 1628 tfm, sizeof(struct caam_request) + 1629 crypto_skcipher_reqsize(fallback)); 1630 } else { 1631 crypto_skcipher_set_reqsize_dma(tfm, 1632 sizeof(struct caam_request)); 1633 } 1634 1635 ret = caam_cra_init(ctx, &caam_alg->caam, false); 1636 if (ret && ctx->fallback) 1637 crypto_free_skcipher(ctx->fallback); 1638 1639 return ret; 1640} 1641 1642static int caam_cra_init_aead(struct crypto_aead *tfm) 1643{ 1644 struct aead_alg *alg = crypto_aead_alg(tfm); 1645 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1646 aead); 1647 1648 crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request)); 1649 return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam, 1650 !caam_alg->caam.nodkp); 1651} 1652 1653static void caam_exit_common(struct caam_ctx *ctx) 1654{ 1655 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1656 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1657 DMA_ATTR_SKIP_CPU_SYNC); 1658} 1659 1660static void caam_cra_exit(struct crypto_skcipher *tfm) 1661{ 1662 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1663 1664 if (ctx->fallback) 1665 crypto_free_skcipher(ctx->fallback); 1666 caam_exit_common(ctx); 1667} 1668 1669static void caam_cra_exit_aead(struct crypto_aead *tfm) 1670{ 1671 caam_exit_common(crypto_aead_ctx_dma(tfm)); 1672} 1673 1674static struct caam_skcipher_alg driver_algs[] = { 1675 { 1676 .skcipher = { 1677 .base = { 1678 .cra_name = "cbc(aes)", 1679 .cra_driver_name = "cbc-aes-caam-qi2", 1680 .cra_blocksize = AES_BLOCK_SIZE, 1681 }, 1682 .setkey = aes_skcipher_setkey, 1683 .encrypt = skcipher_encrypt, 1684 .decrypt = skcipher_decrypt, 1685 .min_keysize = AES_MIN_KEY_SIZE, 1686 .max_keysize = AES_MAX_KEY_SIZE, 1687 .ivsize = AES_BLOCK_SIZE, 1688 }, 1689 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1690 }, 1691 { 1692 .skcipher = { 1693 .base = { 1694 .cra_name = "cbc(des3_ede)", 1695 .cra_driver_name = "cbc-3des-caam-qi2", 1696 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1697 }, 1698 .setkey = des3_skcipher_setkey, 1699 .encrypt = skcipher_encrypt, 1700 .decrypt = skcipher_decrypt, 1701 .min_keysize = DES3_EDE_KEY_SIZE, 1702 .max_keysize = DES3_EDE_KEY_SIZE, 1703 .ivsize = DES3_EDE_BLOCK_SIZE, 1704 }, 1705 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1706 }, 1707 { 1708 .skcipher = { 1709 .base = { 1710 .cra_name = "cbc(des)", 1711 .cra_driver_name = "cbc-des-caam-qi2", 1712 .cra_blocksize = DES_BLOCK_SIZE, 1713 }, 1714 .setkey = des_skcipher_setkey, 1715 .encrypt = skcipher_encrypt, 1716 .decrypt = skcipher_decrypt, 1717 .min_keysize = DES_KEY_SIZE, 1718 .max_keysize = DES_KEY_SIZE, 1719 .ivsize = DES_BLOCK_SIZE, 1720 }, 1721 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1722 }, 1723 { 1724 .skcipher = { 1725 .base = { 1726 .cra_name = "ctr(aes)", 1727 .cra_driver_name = "ctr-aes-caam-qi2", 1728 .cra_blocksize = 1, 1729 }, 1730 .setkey = ctr_skcipher_setkey, 1731 .encrypt = skcipher_encrypt, 1732 .decrypt = skcipher_decrypt, 1733 .min_keysize = AES_MIN_KEY_SIZE, 1734 .max_keysize = AES_MAX_KEY_SIZE, 1735 .ivsize = AES_BLOCK_SIZE, 1736 .chunksize = AES_BLOCK_SIZE, 1737 }, 1738 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1739 OP_ALG_AAI_CTR_MOD128, 1740 }, 1741 { 1742 .skcipher = { 1743 .base = { 1744 .cra_name = "rfc3686(ctr(aes))", 1745 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1746 .cra_blocksize = 1, 1747 }, 1748 .setkey = rfc3686_skcipher_setkey, 1749 .encrypt = skcipher_encrypt, 1750 .decrypt = skcipher_decrypt, 1751 .min_keysize = AES_MIN_KEY_SIZE + 1752 CTR_RFC3686_NONCE_SIZE, 1753 .max_keysize = AES_MAX_KEY_SIZE + 1754 CTR_RFC3686_NONCE_SIZE, 1755 .ivsize = CTR_RFC3686_IV_SIZE, 1756 .chunksize = AES_BLOCK_SIZE, 1757 }, 1758 .caam = { 1759 .class1_alg_type = OP_ALG_ALGSEL_AES | 1760 OP_ALG_AAI_CTR_MOD128, 1761 .rfc3686 = true, 1762 }, 1763 }, 1764 { 1765 .skcipher = { 1766 .base = { 1767 .cra_name = "xts(aes)", 1768 .cra_driver_name = "xts-aes-caam-qi2", 1769 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1770 .cra_blocksize = AES_BLOCK_SIZE, 1771 }, 1772 .setkey = xts_skcipher_setkey, 1773 .encrypt = skcipher_encrypt, 1774 .decrypt = skcipher_decrypt, 1775 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1776 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1777 .ivsize = AES_BLOCK_SIZE, 1778 }, 1779 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1780 }, 1781 { 1782 .skcipher = { 1783 .base = { 1784 .cra_name = "chacha20", 1785 .cra_driver_name = "chacha20-caam-qi2", 1786 .cra_blocksize = 1, 1787 }, 1788 .setkey = chacha20_skcipher_setkey, 1789 .encrypt = skcipher_encrypt, 1790 .decrypt = skcipher_decrypt, 1791 .min_keysize = CHACHA_KEY_SIZE, 1792 .max_keysize = CHACHA_KEY_SIZE, 1793 .ivsize = CHACHA_IV_SIZE, 1794 }, 1795 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1796 }, 1797}; 1798 1799static struct caam_aead_alg driver_aeads[] = { 1800 { 1801 .aead = { 1802 .base = { 1803 .cra_name = "rfc4106(gcm(aes))", 1804 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1805 .cra_blocksize = 1, 1806 }, 1807 .setkey = rfc4106_setkey, 1808 .setauthsize = rfc4106_setauthsize, 1809 .encrypt = ipsec_gcm_encrypt, 1810 .decrypt = ipsec_gcm_decrypt, 1811 .ivsize = 8, 1812 .maxauthsize = AES_BLOCK_SIZE, 1813 }, 1814 .caam = { 1815 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1816 .nodkp = true, 1817 }, 1818 }, 1819 { 1820 .aead = { 1821 .base = { 1822 .cra_name = "rfc4543(gcm(aes))", 1823 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1824 .cra_blocksize = 1, 1825 }, 1826 .setkey = rfc4543_setkey, 1827 .setauthsize = rfc4543_setauthsize, 1828 .encrypt = ipsec_gcm_encrypt, 1829 .decrypt = ipsec_gcm_decrypt, 1830 .ivsize = 8, 1831 .maxauthsize = AES_BLOCK_SIZE, 1832 }, 1833 .caam = { 1834 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1835 .nodkp = true, 1836 }, 1837 }, 1838 /* Galois Counter Mode */ 1839 { 1840 .aead = { 1841 .base = { 1842 .cra_name = "gcm(aes)", 1843 .cra_driver_name = "gcm-aes-caam-qi2", 1844 .cra_blocksize = 1, 1845 }, 1846 .setkey = gcm_setkey, 1847 .setauthsize = gcm_setauthsize, 1848 .encrypt = aead_encrypt, 1849 .decrypt = aead_decrypt, 1850 .ivsize = 12, 1851 .maxauthsize = AES_BLOCK_SIZE, 1852 }, 1853 .caam = { 1854 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1855 .nodkp = true, 1856 } 1857 }, 1858 /* single-pass ipsec_esp descriptor */ 1859 { 1860 .aead = { 1861 .base = { 1862 .cra_name = "authenc(hmac(md5),cbc(aes))", 1863 .cra_driver_name = "authenc-hmac-md5-" 1864 "cbc-aes-caam-qi2", 1865 .cra_blocksize = AES_BLOCK_SIZE, 1866 }, 1867 .setkey = aead_setkey, 1868 .setauthsize = aead_setauthsize, 1869 .encrypt = aead_encrypt, 1870 .decrypt = aead_decrypt, 1871 .ivsize = AES_BLOCK_SIZE, 1872 .maxauthsize = MD5_DIGEST_SIZE, 1873 }, 1874 .caam = { 1875 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1876 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1877 OP_ALG_AAI_HMAC_PRECOMP, 1878 } 1879 }, 1880 { 1881 .aead = { 1882 .base = { 1883 .cra_name = "echainiv(authenc(hmac(md5)," 1884 "cbc(aes)))", 1885 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1886 "cbc-aes-caam-qi2", 1887 .cra_blocksize = AES_BLOCK_SIZE, 1888 }, 1889 .setkey = aead_setkey, 1890 .setauthsize = aead_setauthsize, 1891 .encrypt = aead_encrypt, 1892 .decrypt = aead_decrypt, 1893 .ivsize = AES_BLOCK_SIZE, 1894 .maxauthsize = MD5_DIGEST_SIZE, 1895 }, 1896 .caam = { 1897 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1898 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1899 OP_ALG_AAI_HMAC_PRECOMP, 1900 .geniv = true, 1901 } 1902 }, 1903 { 1904 .aead = { 1905 .base = { 1906 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1907 .cra_driver_name = "authenc-hmac-sha1-" 1908 "cbc-aes-caam-qi2", 1909 .cra_blocksize = AES_BLOCK_SIZE, 1910 }, 1911 .setkey = aead_setkey, 1912 .setauthsize = aead_setauthsize, 1913 .encrypt = aead_encrypt, 1914 .decrypt = aead_decrypt, 1915 .ivsize = AES_BLOCK_SIZE, 1916 .maxauthsize = SHA1_DIGEST_SIZE, 1917 }, 1918 .caam = { 1919 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1920 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1921 OP_ALG_AAI_HMAC_PRECOMP, 1922 } 1923 }, 1924 { 1925 .aead = { 1926 .base = { 1927 .cra_name = "echainiv(authenc(hmac(sha1)," 1928 "cbc(aes)))", 1929 .cra_driver_name = "echainiv-authenc-" 1930 "hmac-sha1-cbc-aes-caam-qi2", 1931 .cra_blocksize = AES_BLOCK_SIZE, 1932 }, 1933 .setkey = aead_setkey, 1934 .setauthsize = aead_setauthsize, 1935 .encrypt = aead_encrypt, 1936 .decrypt = aead_decrypt, 1937 .ivsize = AES_BLOCK_SIZE, 1938 .maxauthsize = SHA1_DIGEST_SIZE, 1939 }, 1940 .caam = { 1941 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1942 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1943 OP_ALG_AAI_HMAC_PRECOMP, 1944 .geniv = true, 1945 }, 1946 }, 1947 { 1948 .aead = { 1949 .base = { 1950 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1951 .cra_driver_name = "authenc-hmac-sha224-" 1952 "cbc-aes-caam-qi2", 1953 .cra_blocksize = AES_BLOCK_SIZE, 1954 }, 1955 .setkey = aead_setkey, 1956 .setauthsize = aead_setauthsize, 1957 .encrypt = aead_encrypt, 1958 .decrypt = aead_decrypt, 1959 .ivsize = AES_BLOCK_SIZE, 1960 .maxauthsize = SHA224_DIGEST_SIZE, 1961 }, 1962 .caam = { 1963 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1964 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1965 OP_ALG_AAI_HMAC_PRECOMP, 1966 } 1967 }, 1968 { 1969 .aead = { 1970 .base = { 1971 .cra_name = "echainiv(authenc(hmac(sha224)," 1972 "cbc(aes)))", 1973 .cra_driver_name = "echainiv-authenc-" 1974 "hmac-sha224-cbc-aes-caam-qi2", 1975 .cra_blocksize = AES_BLOCK_SIZE, 1976 }, 1977 .setkey = aead_setkey, 1978 .setauthsize = aead_setauthsize, 1979 .encrypt = aead_encrypt, 1980 .decrypt = aead_decrypt, 1981 .ivsize = AES_BLOCK_SIZE, 1982 .maxauthsize = SHA224_DIGEST_SIZE, 1983 }, 1984 .caam = { 1985 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1986 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1987 OP_ALG_AAI_HMAC_PRECOMP, 1988 .geniv = true, 1989 } 1990 }, 1991 { 1992 .aead = { 1993 .base = { 1994 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1995 .cra_driver_name = "authenc-hmac-sha256-" 1996 "cbc-aes-caam-qi2", 1997 .cra_blocksize = AES_BLOCK_SIZE, 1998 }, 1999 .setkey = aead_setkey, 2000 .setauthsize = aead_setauthsize, 2001 .encrypt = aead_encrypt, 2002 .decrypt = aead_decrypt, 2003 .ivsize = AES_BLOCK_SIZE, 2004 .maxauthsize = SHA256_DIGEST_SIZE, 2005 }, 2006 .caam = { 2007 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2008 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2009 OP_ALG_AAI_HMAC_PRECOMP, 2010 } 2011 }, 2012 { 2013 .aead = { 2014 .base = { 2015 .cra_name = "echainiv(authenc(hmac(sha256)," 2016 "cbc(aes)))", 2017 .cra_driver_name = "echainiv-authenc-" 2018 "hmac-sha256-cbc-aes-" 2019 "caam-qi2", 2020 .cra_blocksize = AES_BLOCK_SIZE, 2021 }, 2022 .setkey = aead_setkey, 2023 .setauthsize = aead_setauthsize, 2024 .encrypt = aead_encrypt, 2025 .decrypt = aead_decrypt, 2026 .ivsize = AES_BLOCK_SIZE, 2027 .maxauthsize = SHA256_DIGEST_SIZE, 2028 }, 2029 .caam = { 2030 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2031 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2032 OP_ALG_AAI_HMAC_PRECOMP, 2033 .geniv = true, 2034 } 2035 }, 2036 { 2037 .aead = { 2038 .base = { 2039 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2040 .cra_driver_name = "authenc-hmac-sha384-" 2041 "cbc-aes-caam-qi2", 2042 .cra_blocksize = AES_BLOCK_SIZE, 2043 }, 2044 .setkey = aead_setkey, 2045 .setauthsize = aead_setauthsize, 2046 .encrypt = aead_encrypt, 2047 .decrypt = aead_decrypt, 2048 .ivsize = AES_BLOCK_SIZE, 2049 .maxauthsize = SHA384_DIGEST_SIZE, 2050 }, 2051 .caam = { 2052 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2053 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2054 OP_ALG_AAI_HMAC_PRECOMP, 2055 } 2056 }, 2057 { 2058 .aead = { 2059 .base = { 2060 .cra_name = "echainiv(authenc(hmac(sha384)," 2061 "cbc(aes)))", 2062 .cra_driver_name = "echainiv-authenc-" 2063 "hmac-sha384-cbc-aes-" 2064 "caam-qi2", 2065 .cra_blocksize = AES_BLOCK_SIZE, 2066 }, 2067 .setkey = aead_setkey, 2068 .setauthsize = aead_setauthsize, 2069 .encrypt = aead_encrypt, 2070 .decrypt = aead_decrypt, 2071 .ivsize = AES_BLOCK_SIZE, 2072 .maxauthsize = SHA384_DIGEST_SIZE, 2073 }, 2074 .caam = { 2075 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2076 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2077 OP_ALG_AAI_HMAC_PRECOMP, 2078 .geniv = true, 2079 } 2080 }, 2081 { 2082 .aead = { 2083 .base = { 2084 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2085 .cra_driver_name = "authenc-hmac-sha512-" 2086 "cbc-aes-caam-qi2", 2087 .cra_blocksize = AES_BLOCK_SIZE, 2088 }, 2089 .setkey = aead_setkey, 2090 .setauthsize = aead_setauthsize, 2091 .encrypt = aead_encrypt, 2092 .decrypt = aead_decrypt, 2093 .ivsize = AES_BLOCK_SIZE, 2094 .maxauthsize = SHA512_DIGEST_SIZE, 2095 }, 2096 .caam = { 2097 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2098 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2099 OP_ALG_AAI_HMAC_PRECOMP, 2100 } 2101 }, 2102 { 2103 .aead = { 2104 .base = { 2105 .cra_name = "echainiv(authenc(hmac(sha512)," 2106 "cbc(aes)))", 2107 .cra_driver_name = "echainiv-authenc-" 2108 "hmac-sha512-cbc-aes-" 2109 "caam-qi2", 2110 .cra_blocksize = AES_BLOCK_SIZE, 2111 }, 2112 .setkey = aead_setkey, 2113 .setauthsize = aead_setauthsize, 2114 .encrypt = aead_encrypt, 2115 .decrypt = aead_decrypt, 2116 .ivsize = AES_BLOCK_SIZE, 2117 .maxauthsize = SHA512_DIGEST_SIZE, 2118 }, 2119 .caam = { 2120 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2121 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2122 OP_ALG_AAI_HMAC_PRECOMP, 2123 .geniv = true, 2124 } 2125 }, 2126 { 2127 .aead = { 2128 .base = { 2129 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2130 .cra_driver_name = "authenc-hmac-md5-" 2131 "cbc-des3_ede-caam-qi2", 2132 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2133 }, 2134 .setkey = des3_aead_setkey, 2135 .setauthsize = aead_setauthsize, 2136 .encrypt = aead_encrypt, 2137 .decrypt = aead_decrypt, 2138 .ivsize = DES3_EDE_BLOCK_SIZE, 2139 .maxauthsize = MD5_DIGEST_SIZE, 2140 }, 2141 .caam = { 2142 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2143 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2144 OP_ALG_AAI_HMAC_PRECOMP, 2145 } 2146 }, 2147 { 2148 .aead = { 2149 .base = { 2150 .cra_name = "echainiv(authenc(hmac(md5)," 2151 "cbc(des3_ede)))", 2152 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2153 "cbc-des3_ede-caam-qi2", 2154 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2155 }, 2156 .setkey = des3_aead_setkey, 2157 .setauthsize = aead_setauthsize, 2158 .encrypt = aead_encrypt, 2159 .decrypt = aead_decrypt, 2160 .ivsize = DES3_EDE_BLOCK_SIZE, 2161 .maxauthsize = MD5_DIGEST_SIZE, 2162 }, 2163 .caam = { 2164 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2165 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2166 OP_ALG_AAI_HMAC_PRECOMP, 2167 .geniv = true, 2168 } 2169 }, 2170 { 2171 .aead = { 2172 .base = { 2173 .cra_name = "authenc(hmac(sha1)," 2174 "cbc(des3_ede))", 2175 .cra_driver_name = "authenc-hmac-sha1-" 2176 "cbc-des3_ede-caam-qi2", 2177 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2178 }, 2179 .setkey = des3_aead_setkey, 2180 .setauthsize = aead_setauthsize, 2181 .encrypt = aead_encrypt, 2182 .decrypt = aead_decrypt, 2183 .ivsize = DES3_EDE_BLOCK_SIZE, 2184 .maxauthsize = SHA1_DIGEST_SIZE, 2185 }, 2186 .caam = { 2187 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2188 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2189 OP_ALG_AAI_HMAC_PRECOMP, 2190 }, 2191 }, 2192 { 2193 .aead = { 2194 .base = { 2195 .cra_name = "echainiv(authenc(hmac(sha1)," 2196 "cbc(des3_ede)))", 2197 .cra_driver_name = "echainiv-authenc-" 2198 "hmac-sha1-" 2199 "cbc-des3_ede-caam-qi2", 2200 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2201 }, 2202 .setkey = des3_aead_setkey, 2203 .setauthsize = aead_setauthsize, 2204 .encrypt = aead_encrypt, 2205 .decrypt = aead_decrypt, 2206 .ivsize = DES3_EDE_BLOCK_SIZE, 2207 .maxauthsize = SHA1_DIGEST_SIZE, 2208 }, 2209 .caam = { 2210 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2211 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2212 OP_ALG_AAI_HMAC_PRECOMP, 2213 .geniv = true, 2214 } 2215 }, 2216 { 2217 .aead = { 2218 .base = { 2219 .cra_name = "authenc(hmac(sha224)," 2220 "cbc(des3_ede))", 2221 .cra_driver_name = "authenc-hmac-sha224-" 2222 "cbc-des3_ede-caam-qi2", 2223 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2224 }, 2225 .setkey = des3_aead_setkey, 2226 .setauthsize = aead_setauthsize, 2227 .encrypt = aead_encrypt, 2228 .decrypt = aead_decrypt, 2229 .ivsize = DES3_EDE_BLOCK_SIZE, 2230 .maxauthsize = SHA224_DIGEST_SIZE, 2231 }, 2232 .caam = { 2233 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2234 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2235 OP_ALG_AAI_HMAC_PRECOMP, 2236 }, 2237 }, 2238 { 2239 .aead = { 2240 .base = { 2241 .cra_name = "echainiv(authenc(hmac(sha224)," 2242 "cbc(des3_ede)))", 2243 .cra_driver_name = "echainiv-authenc-" 2244 "hmac-sha224-" 2245 "cbc-des3_ede-caam-qi2", 2246 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2247 }, 2248 .setkey = des3_aead_setkey, 2249 .setauthsize = aead_setauthsize, 2250 .encrypt = aead_encrypt, 2251 .decrypt = aead_decrypt, 2252 .ivsize = DES3_EDE_BLOCK_SIZE, 2253 .maxauthsize = SHA224_DIGEST_SIZE, 2254 }, 2255 .caam = { 2256 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2257 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2258 OP_ALG_AAI_HMAC_PRECOMP, 2259 .geniv = true, 2260 } 2261 }, 2262 { 2263 .aead = { 2264 .base = { 2265 .cra_name = "authenc(hmac(sha256)," 2266 "cbc(des3_ede))", 2267 .cra_driver_name = "authenc-hmac-sha256-" 2268 "cbc-des3_ede-caam-qi2", 2269 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2270 }, 2271 .setkey = des3_aead_setkey, 2272 .setauthsize = aead_setauthsize, 2273 .encrypt = aead_encrypt, 2274 .decrypt = aead_decrypt, 2275 .ivsize = DES3_EDE_BLOCK_SIZE, 2276 .maxauthsize = SHA256_DIGEST_SIZE, 2277 }, 2278 .caam = { 2279 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2280 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2281 OP_ALG_AAI_HMAC_PRECOMP, 2282 }, 2283 }, 2284 { 2285 .aead = { 2286 .base = { 2287 .cra_name = "echainiv(authenc(hmac(sha256)," 2288 "cbc(des3_ede)))", 2289 .cra_driver_name = "echainiv-authenc-" 2290 "hmac-sha256-" 2291 "cbc-des3_ede-caam-qi2", 2292 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2293 }, 2294 .setkey = des3_aead_setkey, 2295 .setauthsize = aead_setauthsize, 2296 .encrypt = aead_encrypt, 2297 .decrypt = aead_decrypt, 2298 .ivsize = DES3_EDE_BLOCK_SIZE, 2299 .maxauthsize = SHA256_DIGEST_SIZE, 2300 }, 2301 .caam = { 2302 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2303 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2304 OP_ALG_AAI_HMAC_PRECOMP, 2305 .geniv = true, 2306 } 2307 }, 2308 { 2309 .aead = { 2310 .base = { 2311 .cra_name = "authenc(hmac(sha384)," 2312 "cbc(des3_ede))", 2313 .cra_driver_name = "authenc-hmac-sha384-" 2314 "cbc-des3_ede-caam-qi2", 2315 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2316 }, 2317 .setkey = des3_aead_setkey, 2318 .setauthsize = aead_setauthsize, 2319 .encrypt = aead_encrypt, 2320 .decrypt = aead_decrypt, 2321 .ivsize = DES3_EDE_BLOCK_SIZE, 2322 .maxauthsize = SHA384_DIGEST_SIZE, 2323 }, 2324 .caam = { 2325 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2326 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2327 OP_ALG_AAI_HMAC_PRECOMP, 2328 }, 2329 }, 2330 { 2331 .aead = { 2332 .base = { 2333 .cra_name = "echainiv(authenc(hmac(sha384)," 2334 "cbc(des3_ede)))", 2335 .cra_driver_name = "echainiv-authenc-" 2336 "hmac-sha384-" 2337 "cbc-des3_ede-caam-qi2", 2338 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2339 }, 2340 .setkey = des3_aead_setkey, 2341 .setauthsize = aead_setauthsize, 2342 .encrypt = aead_encrypt, 2343 .decrypt = aead_decrypt, 2344 .ivsize = DES3_EDE_BLOCK_SIZE, 2345 .maxauthsize = SHA384_DIGEST_SIZE, 2346 }, 2347 .caam = { 2348 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2349 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2350 OP_ALG_AAI_HMAC_PRECOMP, 2351 .geniv = true, 2352 } 2353 }, 2354 { 2355 .aead = { 2356 .base = { 2357 .cra_name = "authenc(hmac(sha512)," 2358 "cbc(des3_ede))", 2359 .cra_driver_name = "authenc-hmac-sha512-" 2360 "cbc-des3_ede-caam-qi2", 2361 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2362 }, 2363 .setkey = des3_aead_setkey, 2364 .setauthsize = aead_setauthsize, 2365 .encrypt = aead_encrypt, 2366 .decrypt = aead_decrypt, 2367 .ivsize = DES3_EDE_BLOCK_SIZE, 2368 .maxauthsize = SHA512_DIGEST_SIZE, 2369 }, 2370 .caam = { 2371 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2372 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2373 OP_ALG_AAI_HMAC_PRECOMP, 2374 }, 2375 }, 2376 { 2377 .aead = { 2378 .base = { 2379 .cra_name = "echainiv(authenc(hmac(sha512)," 2380 "cbc(des3_ede)))", 2381 .cra_driver_name = "echainiv-authenc-" 2382 "hmac-sha512-" 2383 "cbc-des3_ede-caam-qi2", 2384 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2385 }, 2386 .setkey = des3_aead_setkey, 2387 .setauthsize = aead_setauthsize, 2388 .encrypt = aead_encrypt, 2389 .decrypt = aead_decrypt, 2390 .ivsize = DES3_EDE_BLOCK_SIZE, 2391 .maxauthsize = SHA512_DIGEST_SIZE, 2392 }, 2393 .caam = { 2394 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2395 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2396 OP_ALG_AAI_HMAC_PRECOMP, 2397 .geniv = true, 2398 } 2399 }, 2400 { 2401 .aead = { 2402 .base = { 2403 .cra_name = "authenc(hmac(md5),cbc(des))", 2404 .cra_driver_name = "authenc-hmac-md5-" 2405 "cbc-des-caam-qi2", 2406 .cra_blocksize = DES_BLOCK_SIZE, 2407 }, 2408 .setkey = aead_setkey, 2409 .setauthsize = aead_setauthsize, 2410 .encrypt = aead_encrypt, 2411 .decrypt = aead_decrypt, 2412 .ivsize = DES_BLOCK_SIZE, 2413 .maxauthsize = MD5_DIGEST_SIZE, 2414 }, 2415 .caam = { 2416 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2417 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2418 OP_ALG_AAI_HMAC_PRECOMP, 2419 }, 2420 }, 2421 { 2422 .aead = { 2423 .base = { 2424 .cra_name = "echainiv(authenc(hmac(md5)," 2425 "cbc(des)))", 2426 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2427 "cbc-des-caam-qi2", 2428 .cra_blocksize = DES_BLOCK_SIZE, 2429 }, 2430 .setkey = aead_setkey, 2431 .setauthsize = aead_setauthsize, 2432 .encrypt = aead_encrypt, 2433 .decrypt = aead_decrypt, 2434 .ivsize = DES_BLOCK_SIZE, 2435 .maxauthsize = MD5_DIGEST_SIZE, 2436 }, 2437 .caam = { 2438 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2439 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2440 OP_ALG_AAI_HMAC_PRECOMP, 2441 .geniv = true, 2442 } 2443 }, 2444 { 2445 .aead = { 2446 .base = { 2447 .cra_name = "authenc(hmac(sha1),cbc(des))", 2448 .cra_driver_name = "authenc-hmac-sha1-" 2449 "cbc-des-caam-qi2", 2450 .cra_blocksize = DES_BLOCK_SIZE, 2451 }, 2452 .setkey = aead_setkey, 2453 .setauthsize = aead_setauthsize, 2454 .encrypt = aead_encrypt, 2455 .decrypt = aead_decrypt, 2456 .ivsize = DES_BLOCK_SIZE, 2457 .maxauthsize = SHA1_DIGEST_SIZE, 2458 }, 2459 .caam = { 2460 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2461 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2462 OP_ALG_AAI_HMAC_PRECOMP, 2463 }, 2464 }, 2465 { 2466 .aead = { 2467 .base = { 2468 .cra_name = "echainiv(authenc(hmac(sha1)," 2469 "cbc(des)))", 2470 .cra_driver_name = "echainiv-authenc-" 2471 "hmac-sha1-cbc-des-caam-qi2", 2472 .cra_blocksize = DES_BLOCK_SIZE, 2473 }, 2474 .setkey = aead_setkey, 2475 .setauthsize = aead_setauthsize, 2476 .encrypt = aead_encrypt, 2477 .decrypt = aead_decrypt, 2478 .ivsize = DES_BLOCK_SIZE, 2479 .maxauthsize = SHA1_DIGEST_SIZE, 2480 }, 2481 .caam = { 2482 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2483 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2484 OP_ALG_AAI_HMAC_PRECOMP, 2485 .geniv = true, 2486 } 2487 }, 2488 { 2489 .aead = { 2490 .base = { 2491 .cra_name = "authenc(hmac(sha224),cbc(des))", 2492 .cra_driver_name = "authenc-hmac-sha224-" 2493 "cbc-des-caam-qi2", 2494 .cra_blocksize = DES_BLOCK_SIZE, 2495 }, 2496 .setkey = aead_setkey, 2497 .setauthsize = aead_setauthsize, 2498 .encrypt = aead_encrypt, 2499 .decrypt = aead_decrypt, 2500 .ivsize = DES_BLOCK_SIZE, 2501 .maxauthsize = SHA224_DIGEST_SIZE, 2502 }, 2503 .caam = { 2504 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2505 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2506 OP_ALG_AAI_HMAC_PRECOMP, 2507 }, 2508 }, 2509 { 2510 .aead = { 2511 .base = { 2512 .cra_name = "echainiv(authenc(hmac(sha224)," 2513 "cbc(des)))", 2514 .cra_driver_name = "echainiv-authenc-" 2515 "hmac-sha224-cbc-des-" 2516 "caam-qi2", 2517 .cra_blocksize = DES_BLOCK_SIZE, 2518 }, 2519 .setkey = aead_setkey, 2520 .setauthsize = aead_setauthsize, 2521 .encrypt = aead_encrypt, 2522 .decrypt = aead_decrypt, 2523 .ivsize = DES_BLOCK_SIZE, 2524 .maxauthsize = SHA224_DIGEST_SIZE, 2525 }, 2526 .caam = { 2527 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2528 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2529 OP_ALG_AAI_HMAC_PRECOMP, 2530 .geniv = true, 2531 } 2532 }, 2533 { 2534 .aead = { 2535 .base = { 2536 .cra_name = "authenc(hmac(sha256),cbc(des))", 2537 .cra_driver_name = "authenc-hmac-sha256-" 2538 "cbc-des-caam-qi2", 2539 .cra_blocksize = DES_BLOCK_SIZE, 2540 }, 2541 .setkey = aead_setkey, 2542 .setauthsize = aead_setauthsize, 2543 .encrypt = aead_encrypt, 2544 .decrypt = aead_decrypt, 2545 .ivsize = DES_BLOCK_SIZE, 2546 .maxauthsize = SHA256_DIGEST_SIZE, 2547 }, 2548 .caam = { 2549 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2550 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2551 OP_ALG_AAI_HMAC_PRECOMP, 2552 }, 2553 }, 2554 { 2555 .aead = { 2556 .base = { 2557 .cra_name = "echainiv(authenc(hmac(sha256)," 2558 "cbc(des)))", 2559 .cra_driver_name = "echainiv-authenc-" 2560 "hmac-sha256-cbc-des-" 2561 "caam-qi2", 2562 .cra_blocksize = DES_BLOCK_SIZE, 2563 }, 2564 .setkey = aead_setkey, 2565 .setauthsize = aead_setauthsize, 2566 .encrypt = aead_encrypt, 2567 .decrypt = aead_decrypt, 2568 .ivsize = DES_BLOCK_SIZE, 2569 .maxauthsize = SHA256_DIGEST_SIZE, 2570 }, 2571 .caam = { 2572 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2573 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2574 OP_ALG_AAI_HMAC_PRECOMP, 2575 .geniv = true, 2576 }, 2577 }, 2578 { 2579 .aead = { 2580 .base = { 2581 .cra_name = "authenc(hmac(sha384),cbc(des))", 2582 .cra_driver_name = "authenc-hmac-sha384-" 2583 "cbc-des-caam-qi2", 2584 .cra_blocksize = DES_BLOCK_SIZE, 2585 }, 2586 .setkey = aead_setkey, 2587 .setauthsize = aead_setauthsize, 2588 .encrypt = aead_encrypt, 2589 .decrypt = aead_decrypt, 2590 .ivsize = DES_BLOCK_SIZE, 2591 .maxauthsize = SHA384_DIGEST_SIZE, 2592 }, 2593 .caam = { 2594 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2595 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2596 OP_ALG_AAI_HMAC_PRECOMP, 2597 }, 2598 }, 2599 { 2600 .aead = { 2601 .base = { 2602 .cra_name = "echainiv(authenc(hmac(sha384)," 2603 "cbc(des)))", 2604 .cra_driver_name = "echainiv-authenc-" 2605 "hmac-sha384-cbc-des-" 2606 "caam-qi2", 2607 .cra_blocksize = DES_BLOCK_SIZE, 2608 }, 2609 .setkey = aead_setkey, 2610 .setauthsize = aead_setauthsize, 2611 .encrypt = aead_encrypt, 2612 .decrypt = aead_decrypt, 2613 .ivsize = DES_BLOCK_SIZE, 2614 .maxauthsize = SHA384_DIGEST_SIZE, 2615 }, 2616 .caam = { 2617 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2618 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2619 OP_ALG_AAI_HMAC_PRECOMP, 2620 .geniv = true, 2621 } 2622 }, 2623 { 2624 .aead = { 2625 .base = { 2626 .cra_name = "authenc(hmac(sha512),cbc(des))", 2627 .cra_driver_name = "authenc-hmac-sha512-" 2628 "cbc-des-caam-qi2", 2629 .cra_blocksize = DES_BLOCK_SIZE, 2630 }, 2631 .setkey = aead_setkey, 2632 .setauthsize = aead_setauthsize, 2633 .encrypt = aead_encrypt, 2634 .decrypt = aead_decrypt, 2635 .ivsize = DES_BLOCK_SIZE, 2636 .maxauthsize = SHA512_DIGEST_SIZE, 2637 }, 2638 .caam = { 2639 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2640 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2641 OP_ALG_AAI_HMAC_PRECOMP, 2642 } 2643 }, 2644 { 2645 .aead = { 2646 .base = { 2647 .cra_name = "echainiv(authenc(hmac(sha512)," 2648 "cbc(des)))", 2649 .cra_driver_name = "echainiv-authenc-" 2650 "hmac-sha512-cbc-des-" 2651 "caam-qi2", 2652 .cra_blocksize = DES_BLOCK_SIZE, 2653 }, 2654 .setkey = aead_setkey, 2655 .setauthsize = aead_setauthsize, 2656 .encrypt = aead_encrypt, 2657 .decrypt = aead_decrypt, 2658 .ivsize = DES_BLOCK_SIZE, 2659 .maxauthsize = SHA512_DIGEST_SIZE, 2660 }, 2661 .caam = { 2662 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2663 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2664 OP_ALG_AAI_HMAC_PRECOMP, 2665 .geniv = true, 2666 } 2667 }, 2668 { 2669 .aead = { 2670 .base = { 2671 .cra_name = "authenc(hmac(md5)," 2672 "rfc3686(ctr(aes)))", 2673 .cra_driver_name = "authenc-hmac-md5-" 2674 "rfc3686-ctr-aes-caam-qi2", 2675 .cra_blocksize = 1, 2676 }, 2677 .setkey = aead_setkey, 2678 .setauthsize = aead_setauthsize, 2679 .encrypt = aead_encrypt, 2680 .decrypt = aead_decrypt, 2681 .ivsize = CTR_RFC3686_IV_SIZE, 2682 .maxauthsize = MD5_DIGEST_SIZE, 2683 }, 2684 .caam = { 2685 .class1_alg_type = OP_ALG_ALGSEL_AES | 2686 OP_ALG_AAI_CTR_MOD128, 2687 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2688 OP_ALG_AAI_HMAC_PRECOMP, 2689 .rfc3686 = true, 2690 }, 2691 }, 2692 { 2693 .aead = { 2694 .base = { 2695 .cra_name = "seqiv(authenc(" 2696 "hmac(md5),rfc3686(ctr(aes))))", 2697 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2698 "rfc3686-ctr-aes-caam-qi2", 2699 .cra_blocksize = 1, 2700 }, 2701 .setkey = aead_setkey, 2702 .setauthsize = aead_setauthsize, 2703 .encrypt = aead_encrypt, 2704 .decrypt = aead_decrypt, 2705 .ivsize = CTR_RFC3686_IV_SIZE, 2706 .maxauthsize = MD5_DIGEST_SIZE, 2707 }, 2708 .caam = { 2709 .class1_alg_type = OP_ALG_ALGSEL_AES | 2710 OP_ALG_AAI_CTR_MOD128, 2711 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2712 OP_ALG_AAI_HMAC_PRECOMP, 2713 .rfc3686 = true, 2714 .geniv = true, 2715 }, 2716 }, 2717 { 2718 .aead = { 2719 .base = { 2720 .cra_name = "authenc(hmac(sha1)," 2721 "rfc3686(ctr(aes)))", 2722 .cra_driver_name = "authenc-hmac-sha1-" 2723 "rfc3686-ctr-aes-caam-qi2", 2724 .cra_blocksize = 1, 2725 }, 2726 .setkey = aead_setkey, 2727 .setauthsize = aead_setauthsize, 2728 .encrypt = aead_encrypt, 2729 .decrypt = aead_decrypt, 2730 .ivsize = CTR_RFC3686_IV_SIZE, 2731 .maxauthsize = SHA1_DIGEST_SIZE, 2732 }, 2733 .caam = { 2734 .class1_alg_type = OP_ALG_ALGSEL_AES | 2735 OP_ALG_AAI_CTR_MOD128, 2736 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2737 OP_ALG_AAI_HMAC_PRECOMP, 2738 .rfc3686 = true, 2739 }, 2740 }, 2741 { 2742 .aead = { 2743 .base = { 2744 .cra_name = "seqiv(authenc(" 2745 "hmac(sha1),rfc3686(ctr(aes))))", 2746 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2747 "rfc3686-ctr-aes-caam-qi2", 2748 .cra_blocksize = 1, 2749 }, 2750 .setkey = aead_setkey, 2751 .setauthsize = aead_setauthsize, 2752 .encrypt = aead_encrypt, 2753 .decrypt = aead_decrypt, 2754 .ivsize = CTR_RFC3686_IV_SIZE, 2755 .maxauthsize = SHA1_DIGEST_SIZE, 2756 }, 2757 .caam = { 2758 .class1_alg_type = OP_ALG_ALGSEL_AES | 2759 OP_ALG_AAI_CTR_MOD128, 2760 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2761 OP_ALG_AAI_HMAC_PRECOMP, 2762 .rfc3686 = true, 2763 .geniv = true, 2764 }, 2765 }, 2766 { 2767 .aead = { 2768 .base = { 2769 .cra_name = "authenc(hmac(sha224)," 2770 "rfc3686(ctr(aes)))", 2771 .cra_driver_name = "authenc-hmac-sha224-" 2772 "rfc3686-ctr-aes-caam-qi2", 2773 .cra_blocksize = 1, 2774 }, 2775 .setkey = aead_setkey, 2776 .setauthsize = aead_setauthsize, 2777 .encrypt = aead_encrypt, 2778 .decrypt = aead_decrypt, 2779 .ivsize = CTR_RFC3686_IV_SIZE, 2780 .maxauthsize = SHA224_DIGEST_SIZE, 2781 }, 2782 .caam = { 2783 .class1_alg_type = OP_ALG_ALGSEL_AES | 2784 OP_ALG_AAI_CTR_MOD128, 2785 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2786 OP_ALG_AAI_HMAC_PRECOMP, 2787 .rfc3686 = true, 2788 }, 2789 }, 2790 { 2791 .aead = { 2792 .base = { 2793 .cra_name = "seqiv(authenc(" 2794 "hmac(sha224),rfc3686(ctr(aes))))", 2795 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2796 "rfc3686-ctr-aes-caam-qi2", 2797 .cra_blocksize = 1, 2798 }, 2799 .setkey = aead_setkey, 2800 .setauthsize = aead_setauthsize, 2801 .encrypt = aead_encrypt, 2802 .decrypt = aead_decrypt, 2803 .ivsize = CTR_RFC3686_IV_SIZE, 2804 .maxauthsize = SHA224_DIGEST_SIZE, 2805 }, 2806 .caam = { 2807 .class1_alg_type = OP_ALG_ALGSEL_AES | 2808 OP_ALG_AAI_CTR_MOD128, 2809 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2810 OP_ALG_AAI_HMAC_PRECOMP, 2811 .rfc3686 = true, 2812 .geniv = true, 2813 }, 2814 }, 2815 { 2816 .aead = { 2817 .base = { 2818 .cra_name = "authenc(hmac(sha256)," 2819 "rfc3686(ctr(aes)))", 2820 .cra_driver_name = "authenc-hmac-sha256-" 2821 "rfc3686-ctr-aes-caam-qi2", 2822 .cra_blocksize = 1, 2823 }, 2824 .setkey = aead_setkey, 2825 .setauthsize = aead_setauthsize, 2826 .encrypt = aead_encrypt, 2827 .decrypt = aead_decrypt, 2828 .ivsize = CTR_RFC3686_IV_SIZE, 2829 .maxauthsize = SHA256_DIGEST_SIZE, 2830 }, 2831 .caam = { 2832 .class1_alg_type = OP_ALG_ALGSEL_AES | 2833 OP_ALG_AAI_CTR_MOD128, 2834 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2835 OP_ALG_AAI_HMAC_PRECOMP, 2836 .rfc3686 = true, 2837 }, 2838 }, 2839 { 2840 .aead = { 2841 .base = { 2842 .cra_name = "seqiv(authenc(hmac(sha256)," 2843 "rfc3686(ctr(aes))))", 2844 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2845 "rfc3686-ctr-aes-caam-qi2", 2846 .cra_blocksize = 1, 2847 }, 2848 .setkey = aead_setkey, 2849 .setauthsize = aead_setauthsize, 2850 .encrypt = aead_encrypt, 2851 .decrypt = aead_decrypt, 2852 .ivsize = CTR_RFC3686_IV_SIZE, 2853 .maxauthsize = SHA256_DIGEST_SIZE, 2854 }, 2855 .caam = { 2856 .class1_alg_type = OP_ALG_ALGSEL_AES | 2857 OP_ALG_AAI_CTR_MOD128, 2858 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2859 OP_ALG_AAI_HMAC_PRECOMP, 2860 .rfc3686 = true, 2861 .geniv = true, 2862 }, 2863 }, 2864 { 2865 .aead = { 2866 .base = { 2867 .cra_name = "authenc(hmac(sha384)," 2868 "rfc3686(ctr(aes)))", 2869 .cra_driver_name = "authenc-hmac-sha384-" 2870 "rfc3686-ctr-aes-caam-qi2", 2871 .cra_blocksize = 1, 2872 }, 2873 .setkey = aead_setkey, 2874 .setauthsize = aead_setauthsize, 2875 .encrypt = aead_encrypt, 2876 .decrypt = aead_decrypt, 2877 .ivsize = CTR_RFC3686_IV_SIZE, 2878 .maxauthsize = SHA384_DIGEST_SIZE, 2879 }, 2880 .caam = { 2881 .class1_alg_type = OP_ALG_ALGSEL_AES | 2882 OP_ALG_AAI_CTR_MOD128, 2883 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2884 OP_ALG_AAI_HMAC_PRECOMP, 2885 .rfc3686 = true, 2886 }, 2887 }, 2888 { 2889 .aead = { 2890 .base = { 2891 .cra_name = "seqiv(authenc(hmac(sha384)," 2892 "rfc3686(ctr(aes))))", 2893 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2894 "rfc3686-ctr-aes-caam-qi2", 2895 .cra_blocksize = 1, 2896 }, 2897 .setkey = aead_setkey, 2898 .setauthsize = aead_setauthsize, 2899 .encrypt = aead_encrypt, 2900 .decrypt = aead_decrypt, 2901 .ivsize = CTR_RFC3686_IV_SIZE, 2902 .maxauthsize = SHA384_DIGEST_SIZE, 2903 }, 2904 .caam = { 2905 .class1_alg_type = OP_ALG_ALGSEL_AES | 2906 OP_ALG_AAI_CTR_MOD128, 2907 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2908 OP_ALG_AAI_HMAC_PRECOMP, 2909 .rfc3686 = true, 2910 .geniv = true, 2911 }, 2912 }, 2913 { 2914 .aead = { 2915 .base = { 2916 .cra_name = "rfc7539(chacha20,poly1305)", 2917 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2918 "caam-qi2", 2919 .cra_blocksize = 1, 2920 }, 2921 .setkey = chachapoly_setkey, 2922 .setauthsize = chachapoly_setauthsize, 2923 .encrypt = aead_encrypt, 2924 .decrypt = aead_decrypt, 2925 .ivsize = CHACHAPOLY_IV_SIZE, 2926 .maxauthsize = POLY1305_DIGEST_SIZE, 2927 }, 2928 .caam = { 2929 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2930 OP_ALG_AAI_AEAD, 2931 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2932 OP_ALG_AAI_AEAD, 2933 .nodkp = true, 2934 }, 2935 }, 2936 { 2937 .aead = { 2938 .base = { 2939 .cra_name = "rfc7539esp(chacha20,poly1305)", 2940 .cra_driver_name = "rfc7539esp-chacha20-" 2941 "poly1305-caam-qi2", 2942 .cra_blocksize = 1, 2943 }, 2944 .setkey = chachapoly_setkey, 2945 .setauthsize = chachapoly_setauthsize, 2946 .encrypt = aead_encrypt, 2947 .decrypt = aead_decrypt, 2948 .ivsize = 8, 2949 .maxauthsize = POLY1305_DIGEST_SIZE, 2950 }, 2951 .caam = { 2952 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2953 OP_ALG_AAI_AEAD, 2954 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2955 OP_ALG_AAI_AEAD, 2956 .nodkp = true, 2957 }, 2958 }, 2959 { 2960 .aead = { 2961 .base = { 2962 .cra_name = "authenc(hmac(sha512)," 2963 "rfc3686(ctr(aes)))", 2964 .cra_driver_name = "authenc-hmac-sha512-" 2965 "rfc3686-ctr-aes-caam-qi2", 2966 .cra_blocksize = 1, 2967 }, 2968 .setkey = aead_setkey, 2969 .setauthsize = aead_setauthsize, 2970 .encrypt = aead_encrypt, 2971 .decrypt = aead_decrypt, 2972 .ivsize = CTR_RFC3686_IV_SIZE, 2973 .maxauthsize = SHA512_DIGEST_SIZE, 2974 }, 2975 .caam = { 2976 .class1_alg_type = OP_ALG_ALGSEL_AES | 2977 OP_ALG_AAI_CTR_MOD128, 2978 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2979 OP_ALG_AAI_HMAC_PRECOMP, 2980 .rfc3686 = true, 2981 }, 2982 }, 2983 { 2984 .aead = { 2985 .base = { 2986 .cra_name = "seqiv(authenc(hmac(sha512)," 2987 "rfc3686(ctr(aes))))", 2988 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2989 "rfc3686-ctr-aes-caam-qi2", 2990 .cra_blocksize = 1, 2991 }, 2992 .setkey = aead_setkey, 2993 .setauthsize = aead_setauthsize, 2994 .encrypt = aead_encrypt, 2995 .decrypt = aead_decrypt, 2996 .ivsize = CTR_RFC3686_IV_SIZE, 2997 .maxauthsize = SHA512_DIGEST_SIZE, 2998 }, 2999 .caam = { 3000 .class1_alg_type = OP_ALG_ALGSEL_AES | 3001 OP_ALG_AAI_CTR_MOD128, 3002 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3003 OP_ALG_AAI_HMAC_PRECOMP, 3004 .rfc3686 = true, 3005 .geniv = true, 3006 }, 3007 }, 3008}; 3009 3010static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3011{ 3012 struct skcipher_alg *alg = &t_alg->skcipher; 3013 3014 alg->base.cra_module = THIS_MODULE; 3015 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3016 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3017 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3018 CRYPTO_ALG_KERN_DRIVER_ONLY); 3019 3020 alg->init = caam_cra_init_skcipher; 3021 alg->exit = caam_cra_exit; 3022} 3023 3024static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3025{ 3026 struct aead_alg *alg = &t_alg->aead; 3027 3028 alg->base.cra_module = THIS_MODULE; 3029 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3030 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3031 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3032 CRYPTO_ALG_KERN_DRIVER_ONLY; 3033 3034 alg->init = caam_cra_init_aead; 3035 alg->exit = caam_cra_exit_aead; 3036} 3037 3038/* max hash key is max split key size */ 3039#define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 3040 3041#define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 3042 3043/* caam context sizes for hashes: running digest + 8 */ 3044#define HASH_MSG_LEN 8 3045#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 3046 3047enum hash_optype { 3048 UPDATE = 0, 3049 UPDATE_FIRST, 3050 FINALIZE, 3051 DIGEST, 3052 HASH_NUM_OP 3053}; 3054 3055/** 3056 * struct caam_hash_ctx - ahash per-session context 3057 * @flc: Flow Contexts array 3058 * @key: authentication key 3059 * @flc_dma: I/O virtual addresses of the Flow Contexts 3060 * @dev: dpseci device 3061 * @ctx_len: size of Context Register 3062 * @adata: hashing algorithm details 3063 */ 3064struct caam_hash_ctx { 3065 struct caam_flc flc[HASH_NUM_OP]; 3066 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3067 dma_addr_t flc_dma[HASH_NUM_OP]; 3068 struct device *dev; 3069 int ctx_len; 3070 struct alginfo adata; 3071}; 3072 3073/* ahash state */ 3074struct caam_hash_state { 3075 struct caam_request caam_req; 3076 dma_addr_t buf_dma; 3077 dma_addr_t ctx_dma; 3078 int ctx_dma_len; 3079 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3080 int buflen; 3081 int next_buflen; 3082 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3083 int (*update)(struct ahash_request *req); 3084 int (*final)(struct ahash_request *req); 3085 int (*finup)(struct ahash_request *req); 3086}; 3087 3088struct caam_export_state { 3089 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3090 u8 caam_ctx[MAX_CTX_LEN]; 3091 int buflen; 3092 int (*update)(struct ahash_request *req); 3093 int (*final)(struct ahash_request *req); 3094 int (*finup)(struct ahash_request *req); 3095}; 3096 3097/* Map current buffer in state (if length > 0) and put it in link table */ 3098static inline int buf_map_to_qm_sg(struct device *dev, 3099 struct dpaa2_sg_entry *qm_sg, 3100 struct caam_hash_state *state) 3101{ 3102 int buflen = state->buflen; 3103 3104 if (!buflen) 3105 return 0; 3106 3107 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3108 DMA_TO_DEVICE); 3109 if (dma_mapping_error(dev, state->buf_dma)) { 3110 dev_err(dev, "unable to map buf\n"); 3111 state->buf_dma = 0; 3112 return -ENOMEM; 3113 } 3114 3115 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3116 3117 return 0; 3118} 3119 3120/* Map state->caam_ctx, and add it to link table */ 3121static inline int ctx_map_to_qm_sg(struct device *dev, 3122 struct caam_hash_state *state, int ctx_len, 3123 struct dpaa2_sg_entry *qm_sg, u32 flag) 3124{ 3125 state->ctx_dma_len = ctx_len; 3126 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3127 if (dma_mapping_error(dev, state->ctx_dma)) { 3128 dev_err(dev, "unable to map ctx\n"); 3129 state->ctx_dma = 0; 3130 return -ENOMEM; 3131 } 3132 3133 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3134 3135 return 0; 3136} 3137 3138static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3139{ 3140 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3141 int digestsize = crypto_ahash_digestsize(ahash); 3142 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3143 struct caam_flc *flc; 3144 u32 *desc; 3145 3146 /* ahash_update shared descriptor */ 3147 flc = &ctx->flc[UPDATE]; 3148 desc = flc->sh_desc; 3149 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3150 ctx->ctx_len, true, priv->sec_attr.era); 3151 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3152 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3153 desc_bytes(desc), DMA_BIDIRECTIONAL); 3154 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3155 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3156 1); 3157 3158 /* ahash_update_first shared descriptor */ 3159 flc = &ctx->flc[UPDATE_FIRST]; 3160 desc = flc->sh_desc; 3161 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3162 ctx->ctx_len, false, priv->sec_attr.era); 3163 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3164 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3165 desc_bytes(desc), DMA_BIDIRECTIONAL); 3166 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3167 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3168 1); 3169 3170 /* ahash_final shared descriptor */ 3171 flc = &ctx->flc[FINALIZE]; 3172 desc = flc->sh_desc; 3173 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3174 ctx->ctx_len, true, priv->sec_attr.era); 3175 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3176 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3177 desc_bytes(desc), DMA_BIDIRECTIONAL); 3178 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3179 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3180 1); 3181 3182 /* ahash_digest shared descriptor */ 3183 flc = &ctx->flc[DIGEST]; 3184 desc = flc->sh_desc; 3185 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3186 ctx->ctx_len, false, priv->sec_attr.era); 3187 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3188 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3189 desc_bytes(desc), DMA_BIDIRECTIONAL); 3190 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3191 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3192 1); 3193 3194 return 0; 3195} 3196 3197struct split_key_sh_result { 3198 struct completion completion; 3199 int err; 3200 struct device *dev; 3201}; 3202 3203static void split_key_sh_done(void *cbk_ctx, u32 err) 3204{ 3205 struct split_key_sh_result *res = cbk_ctx; 3206 3207 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3208 3209 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3210 complete(&res->completion); 3211} 3212 3213/* Digest hash size if it is too large */ 3214static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3215 u32 digestsize) 3216{ 3217 struct caam_request *req_ctx; 3218 u32 *desc; 3219 struct split_key_sh_result result; 3220 dma_addr_t key_dma; 3221 struct caam_flc *flc; 3222 dma_addr_t flc_dma; 3223 int ret = -ENOMEM; 3224 struct dpaa2_fl_entry *in_fle, *out_fle; 3225 3226 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL); 3227 if (!req_ctx) 3228 return -ENOMEM; 3229 3230 in_fle = &req_ctx->fd_flt[1]; 3231 out_fle = &req_ctx->fd_flt[0]; 3232 3233 flc = kzalloc(sizeof(*flc), GFP_KERNEL); 3234 if (!flc) 3235 goto err_flc; 3236 3237 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3238 if (dma_mapping_error(ctx->dev, key_dma)) { 3239 dev_err(ctx->dev, "unable to map key memory\n"); 3240 goto err_key_dma; 3241 } 3242 3243 desc = flc->sh_desc; 3244 3245 init_sh_desc(desc, 0); 3246 3247 /* descriptor to perform unkeyed hash on key_in */ 3248 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3249 OP_ALG_AS_INITFINAL); 3250 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3251 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3252 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3253 LDST_SRCDST_BYTE_CONTEXT); 3254 3255 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3256 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3257 desc_bytes(desc), DMA_TO_DEVICE); 3258 if (dma_mapping_error(ctx->dev, flc_dma)) { 3259 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3260 goto err_flc_dma; 3261 } 3262 3263 dpaa2_fl_set_final(in_fle, true); 3264 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3265 dpaa2_fl_set_addr(in_fle, key_dma); 3266 dpaa2_fl_set_len(in_fle, *keylen); 3267 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3268 dpaa2_fl_set_addr(out_fle, key_dma); 3269 dpaa2_fl_set_len(out_fle, digestsize); 3270 3271 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3272 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3273 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3274 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3275 1); 3276 3277 result.err = 0; 3278 init_completion(&result.completion); 3279 result.dev = ctx->dev; 3280 3281 req_ctx->flc = flc; 3282 req_ctx->flc_dma = flc_dma; 3283 req_ctx->cbk = split_key_sh_done; 3284 req_ctx->ctx = &result; 3285 3286 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3287 if (ret == -EINPROGRESS) { 3288 /* in progress */ 3289 wait_for_completion(&result.completion); 3290 ret = result.err; 3291 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3292 DUMP_PREFIX_ADDRESS, 16, 4, key, 3293 digestsize, 1); 3294 } 3295 3296 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3297 DMA_TO_DEVICE); 3298err_flc_dma: 3299 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3300err_key_dma: 3301 kfree(flc); 3302err_flc: 3303 kfree(req_ctx); 3304 3305 *keylen = digestsize; 3306 3307 return ret; 3308} 3309 3310static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3311 unsigned int keylen) 3312{ 3313 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3314 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3315 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3316 int ret; 3317 u8 *hashed_key = NULL; 3318 3319 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3320 3321 if (keylen > blocksize) { 3322 unsigned int aligned_len = 3323 ALIGN(keylen, dma_get_cache_alignment()); 3324 3325 if (aligned_len < keylen) 3326 return -EOVERFLOW; 3327 3328 hashed_key = kmemdup(key, aligned_len, GFP_KERNEL); 3329 if (!hashed_key) 3330 return -ENOMEM; 3331 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3332 if (ret) 3333 goto bad_free_key; 3334 key = hashed_key; 3335 } 3336 3337 ctx->adata.keylen = keylen; 3338 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3339 OP_ALG_ALGSEL_MASK); 3340 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3341 goto bad_free_key; 3342 3343 ctx->adata.key_virt = key; 3344 ctx->adata.key_inline = true; 3345 3346 /* 3347 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3348 * in invalid opcodes (last bytes of user key) in the resulting 3349 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3350 * addresses are needed. 3351 */ 3352 if (keylen > ctx->adata.keylen_pad) { 3353 memcpy(ctx->key, key, keylen); 3354 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3355 ctx->adata.keylen_pad, 3356 DMA_TO_DEVICE); 3357 } 3358 3359 ret = ahash_set_sh_desc(ahash); 3360 kfree(hashed_key); 3361 return ret; 3362bad_free_key: 3363 kfree(hashed_key); 3364 return -EINVAL; 3365} 3366 3367static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3368 struct ahash_request *req) 3369{ 3370 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3371 3372 if (edesc->src_nents) 3373 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3374 3375 if (edesc->qm_sg_bytes) 3376 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3377 DMA_TO_DEVICE); 3378 3379 if (state->buf_dma) { 3380 dma_unmap_single(dev, state->buf_dma, state->buflen, 3381 DMA_TO_DEVICE); 3382 state->buf_dma = 0; 3383 } 3384} 3385 3386static inline void ahash_unmap_ctx(struct device *dev, 3387 struct ahash_edesc *edesc, 3388 struct ahash_request *req, u32 flag) 3389{ 3390 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3391 3392 if (state->ctx_dma) { 3393 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3394 state->ctx_dma = 0; 3395 } 3396 ahash_unmap(dev, edesc, req); 3397} 3398 3399static void ahash_done(void *cbk_ctx, u32 status) 3400{ 3401 struct crypto_async_request *areq = cbk_ctx; 3402 struct ahash_request *req = ahash_request_cast(areq); 3403 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3404 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3405 struct ahash_edesc *edesc = state->caam_req.edesc; 3406 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3407 int digestsize = crypto_ahash_digestsize(ahash); 3408 int ecode = 0; 3409 3410 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3411 3412 if (unlikely(status)) 3413 ecode = caam_qi2_strstatus(ctx->dev, status); 3414 3415 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3416 memcpy(req->result, state->caam_ctx, digestsize); 3417 qi_cache_free(edesc); 3418 3419 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3420 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3421 ctx->ctx_len, 1); 3422 3423 ahash_request_complete(req, ecode); 3424} 3425 3426static void ahash_done_bi(void *cbk_ctx, u32 status) 3427{ 3428 struct crypto_async_request *areq = cbk_ctx; 3429 struct ahash_request *req = ahash_request_cast(areq); 3430 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3431 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3432 struct ahash_edesc *edesc = state->caam_req.edesc; 3433 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3434 int ecode = 0; 3435 3436 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3437 3438 if (unlikely(status)) 3439 ecode = caam_qi2_strstatus(ctx->dev, status); 3440 3441 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3442 qi_cache_free(edesc); 3443 3444 scatterwalk_map_and_copy(state->buf, req->src, 3445 req->nbytes - state->next_buflen, 3446 state->next_buflen, 0); 3447 state->buflen = state->next_buflen; 3448 3449 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3450 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3451 state->buflen, 1); 3452 3453 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3454 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3455 ctx->ctx_len, 1); 3456 if (req->result) 3457 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3458 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3459 crypto_ahash_digestsize(ahash), 1); 3460 3461 ahash_request_complete(req, ecode); 3462} 3463 3464static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3465{ 3466 struct crypto_async_request *areq = cbk_ctx; 3467 struct ahash_request *req = ahash_request_cast(areq); 3468 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3469 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3470 struct ahash_edesc *edesc = state->caam_req.edesc; 3471 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3472 int digestsize = crypto_ahash_digestsize(ahash); 3473 int ecode = 0; 3474 3475 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3476 3477 if (unlikely(status)) 3478 ecode = caam_qi2_strstatus(ctx->dev, status); 3479 3480 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3481 memcpy(req->result, state->caam_ctx, digestsize); 3482 qi_cache_free(edesc); 3483 3484 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3485 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3486 ctx->ctx_len, 1); 3487 3488 ahash_request_complete(req, ecode); 3489} 3490 3491static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3492{ 3493 struct crypto_async_request *areq = cbk_ctx; 3494 struct ahash_request *req = ahash_request_cast(areq); 3495 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3496 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3497 struct ahash_edesc *edesc = state->caam_req.edesc; 3498 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3499 int ecode = 0; 3500 3501 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3502 3503 if (unlikely(status)) 3504 ecode = caam_qi2_strstatus(ctx->dev, status); 3505 3506 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3507 qi_cache_free(edesc); 3508 3509 scatterwalk_map_and_copy(state->buf, req->src, 3510 req->nbytes - state->next_buflen, 3511 state->next_buflen, 0); 3512 state->buflen = state->next_buflen; 3513 3514 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3515 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3516 state->buflen, 1); 3517 3518 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3519 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3520 ctx->ctx_len, 1); 3521 if (req->result) 3522 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3523 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3524 crypto_ahash_digestsize(ahash), 1); 3525 3526 ahash_request_complete(req, ecode); 3527} 3528 3529static int ahash_update_ctx(struct ahash_request *req) 3530{ 3531 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3532 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3533 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3534 struct caam_request *req_ctx = &state->caam_req; 3535 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3536 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3537 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3538 GFP_KERNEL : GFP_ATOMIC; 3539 u8 *buf = state->buf; 3540 int *buflen = &state->buflen; 3541 int *next_buflen = &state->next_buflen; 3542 int in_len = *buflen + req->nbytes, to_hash; 3543 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3544 struct ahash_edesc *edesc; 3545 int ret = 0; 3546 3547 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3548 to_hash = in_len - *next_buflen; 3549 3550 if (to_hash) { 3551 struct dpaa2_sg_entry *sg_table; 3552 int src_len = req->nbytes - *next_buflen; 3553 3554 src_nents = sg_nents_for_len(req->src, src_len); 3555 if (src_nents < 0) { 3556 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3557 return src_nents; 3558 } 3559 3560 if (src_nents) { 3561 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3562 DMA_TO_DEVICE); 3563 if (!mapped_nents) { 3564 dev_err(ctx->dev, "unable to DMA map source\n"); 3565 return -ENOMEM; 3566 } 3567 } else { 3568 mapped_nents = 0; 3569 } 3570 3571 /* allocate space for base edesc and link tables */ 3572 edesc = qi_cache_zalloc(flags); 3573 if (!edesc) { 3574 dma_unmap_sg(ctx->dev, req->src, src_nents, 3575 DMA_TO_DEVICE); 3576 return -ENOMEM; 3577 } 3578 3579 edesc->src_nents = src_nents; 3580 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3581 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3582 sizeof(*sg_table); 3583 sg_table = &edesc->sgt[0]; 3584 3585 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3586 DMA_BIDIRECTIONAL); 3587 if (ret) 3588 goto unmap_ctx; 3589 3590 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3591 if (ret) 3592 goto unmap_ctx; 3593 3594 if (mapped_nents) { 3595 sg_to_qm_sg_last(req->src, src_len, 3596 sg_table + qm_sg_src_index, 0); 3597 } else { 3598 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3599 true); 3600 } 3601 3602 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3603 qm_sg_bytes, DMA_TO_DEVICE); 3604 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3605 dev_err(ctx->dev, "unable to map S/G table\n"); 3606 ret = -ENOMEM; 3607 goto unmap_ctx; 3608 } 3609 edesc->qm_sg_bytes = qm_sg_bytes; 3610 3611 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3612 dpaa2_fl_set_final(in_fle, true); 3613 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3614 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3615 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3616 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3617 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3618 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3619 3620 req_ctx->flc = &ctx->flc[UPDATE]; 3621 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3622 req_ctx->cbk = ahash_done_bi; 3623 req_ctx->ctx = &req->base; 3624 req_ctx->edesc = edesc; 3625 3626 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3627 if (ret != -EINPROGRESS && 3628 !(ret == -EBUSY && 3629 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3630 goto unmap_ctx; 3631 } else if (*next_buflen) { 3632 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3633 req->nbytes, 0); 3634 *buflen = *next_buflen; 3635 3636 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3637 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3638 *buflen, 1); 3639 } 3640 3641 return ret; 3642unmap_ctx: 3643 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3644 qi_cache_free(edesc); 3645 return ret; 3646} 3647 3648static int ahash_final_ctx(struct ahash_request *req) 3649{ 3650 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3651 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3652 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3653 struct caam_request *req_ctx = &state->caam_req; 3654 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3655 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3656 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3657 GFP_KERNEL : GFP_ATOMIC; 3658 int buflen = state->buflen; 3659 int qm_sg_bytes; 3660 int digestsize = crypto_ahash_digestsize(ahash); 3661 struct ahash_edesc *edesc; 3662 struct dpaa2_sg_entry *sg_table; 3663 int ret; 3664 3665 /* allocate space for base edesc and link tables */ 3666 edesc = qi_cache_zalloc(flags); 3667 if (!edesc) 3668 return -ENOMEM; 3669 3670 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3671 sg_table = &edesc->sgt[0]; 3672 3673 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3674 DMA_BIDIRECTIONAL); 3675 if (ret) 3676 goto unmap_ctx; 3677 3678 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3679 if (ret) 3680 goto unmap_ctx; 3681 3682 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3683 3684 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3685 DMA_TO_DEVICE); 3686 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3687 dev_err(ctx->dev, "unable to map S/G table\n"); 3688 ret = -ENOMEM; 3689 goto unmap_ctx; 3690 } 3691 edesc->qm_sg_bytes = qm_sg_bytes; 3692 3693 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3694 dpaa2_fl_set_final(in_fle, true); 3695 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3696 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3697 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3698 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3699 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3700 dpaa2_fl_set_len(out_fle, digestsize); 3701 3702 req_ctx->flc = &ctx->flc[FINALIZE]; 3703 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3704 req_ctx->cbk = ahash_done_ctx_src; 3705 req_ctx->ctx = &req->base; 3706 req_ctx->edesc = edesc; 3707 3708 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3709 if (ret == -EINPROGRESS || 3710 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3711 return ret; 3712 3713unmap_ctx: 3714 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3715 qi_cache_free(edesc); 3716 return ret; 3717} 3718 3719static int ahash_finup_ctx(struct ahash_request *req) 3720{ 3721 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3722 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3723 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3724 struct caam_request *req_ctx = &state->caam_req; 3725 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3726 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3727 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3728 GFP_KERNEL : GFP_ATOMIC; 3729 int buflen = state->buflen; 3730 int qm_sg_bytes, qm_sg_src_index; 3731 int src_nents, mapped_nents; 3732 int digestsize = crypto_ahash_digestsize(ahash); 3733 struct ahash_edesc *edesc; 3734 struct dpaa2_sg_entry *sg_table; 3735 int ret; 3736 3737 src_nents = sg_nents_for_len(req->src, req->nbytes); 3738 if (src_nents < 0) { 3739 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3740 return src_nents; 3741 } 3742 3743 if (src_nents) { 3744 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3745 DMA_TO_DEVICE); 3746 if (!mapped_nents) { 3747 dev_err(ctx->dev, "unable to DMA map source\n"); 3748 return -ENOMEM; 3749 } 3750 } else { 3751 mapped_nents = 0; 3752 } 3753 3754 /* allocate space for base edesc and link tables */ 3755 edesc = qi_cache_zalloc(flags); 3756 if (!edesc) { 3757 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3758 return -ENOMEM; 3759 } 3760 3761 edesc->src_nents = src_nents; 3762 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3763 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3764 sizeof(*sg_table); 3765 sg_table = &edesc->sgt[0]; 3766 3767 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3768 DMA_BIDIRECTIONAL); 3769 if (ret) 3770 goto unmap_ctx; 3771 3772 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3773 if (ret) 3774 goto unmap_ctx; 3775 3776 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3777 3778 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3779 DMA_TO_DEVICE); 3780 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3781 dev_err(ctx->dev, "unable to map S/G table\n"); 3782 ret = -ENOMEM; 3783 goto unmap_ctx; 3784 } 3785 edesc->qm_sg_bytes = qm_sg_bytes; 3786 3787 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3788 dpaa2_fl_set_final(in_fle, true); 3789 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3790 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3791 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3792 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3793 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3794 dpaa2_fl_set_len(out_fle, digestsize); 3795 3796 req_ctx->flc = &ctx->flc[FINALIZE]; 3797 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3798 req_ctx->cbk = ahash_done_ctx_src; 3799 req_ctx->ctx = &req->base; 3800 req_ctx->edesc = edesc; 3801 3802 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3803 if (ret == -EINPROGRESS || 3804 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3805 return ret; 3806 3807unmap_ctx: 3808 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3809 qi_cache_free(edesc); 3810 return ret; 3811} 3812 3813static int ahash_digest(struct ahash_request *req) 3814{ 3815 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3816 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3817 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3818 struct caam_request *req_ctx = &state->caam_req; 3819 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3820 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3821 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3822 GFP_KERNEL : GFP_ATOMIC; 3823 int digestsize = crypto_ahash_digestsize(ahash); 3824 int src_nents, mapped_nents; 3825 struct ahash_edesc *edesc; 3826 int ret = -ENOMEM; 3827 3828 state->buf_dma = 0; 3829 3830 src_nents = sg_nents_for_len(req->src, req->nbytes); 3831 if (src_nents < 0) { 3832 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3833 return src_nents; 3834 } 3835 3836 if (src_nents) { 3837 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3838 DMA_TO_DEVICE); 3839 if (!mapped_nents) { 3840 dev_err(ctx->dev, "unable to map source for DMA\n"); 3841 return ret; 3842 } 3843 } else { 3844 mapped_nents = 0; 3845 } 3846 3847 /* allocate space for base edesc and link tables */ 3848 edesc = qi_cache_zalloc(flags); 3849 if (!edesc) { 3850 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3851 return ret; 3852 } 3853 3854 edesc->src_nents = src_nents; 3855 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3856 3857 if (mapped_nents > 1) { 3858 int qm_sg_bytes; 3859 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3860 3861 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3862 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3863 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3864 qm_sg_bytes, DMA_TO_DEVICE); 3865 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3866 dev_err(ctx->dev, "unable to map S/G table\n"); 3867 goto unmap; 3868 } 3869 edesc->qm_sg_bytes = qm_sg_bytes; 3870 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3871 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3872 } else { 3873 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3874 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3875 } 3876 3877 state->ctx_dma_len = digestsize; 3878 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3879 DMA_FROM_DEVICE); 3880 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3881 dev_err(ctx->dev, "unable to map ctx\n"); 3882 state->ctx_dma = 0; 3883 goto unmap; 3884 } 3885 3886 dpaa2_fl_set_final(in_fle, true); 3887 dpaa2_fl_set_len(in_fle, req->nbytes); 3888 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3889 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3890 dpaa2_fl_set_len(out_fle, digestsize); 3891 3892 req_ctx->flc = &ctx->flc[DIGEST]; 3893 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3894 req_ctx->cbk = ahash_done; 3895 req_ctx->ctx = &req->base; 3896 req_ctx->edesc = edesc; 3897 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3898 if (ret == -EINPROGRESS || 3899 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3900 return ret; 3901 3902unmap: 3903 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3904 qi_cache_free(edesc); 3905 return ret; 3906} 3907 3908static int ahash_final_no_ctx(struct ahash_request *req) 3909{ 3910 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3911 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3912 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3913 struct caam_request *req_ctx = &state->caam_req; 3914 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3915 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3916 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3917 GFP_KERNEL : GFP_ATOMIC; 3918 u8 *buf = state->buf; 3919 int buflen = state->buflen; 3920 int digestsize = crypto_ahash_digestsize(ahash); 3921 struct ahash_edesc *edesc; 3922 int ret = -ENOMEM; 3923 3924 /* allocate space for base edesc and link tables */ 3925 edesc = qi_cache_zalloc(flags); 3926 if (!edesc) 3927 return ret; 3928 3929 if (buflen) { 3930 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3931 DMA_TO_DEVICE); 3932 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3933 dev_err(ctx->dev, "unable to map src\n"); 3934 goto unmap; 3935 } 3936 } 3937 3938 state->ctx_dma_len = digestsize; 3939 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3940 DMA_FROM_DEVICE); 3941 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3942 dev_err(ctx->dev, "unable to map ctx\n"); 3943 state->ctx_dma = 0; 3944 goto unmap; 3945 } 3946 3947 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3948 dpaa2_fl_set_final(in_fle, true); 3949 /* 3950 * crypto engine requires the input entry to be present when 3951 * "frame list" FD is used. 3952 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3953 * in_fle zeroized (except for "Final" flag) is the best option. 3954 */ 3955 if (buflen) { 3956 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3957 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3958 dpaa2_fl_set_len(in_fle, buflen); 3959 } 3960 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3961 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3962 dpaa2_fl_set_len(out_fle, digestsize); 3963 3964 req_ctx->flc = &ctx->flc[DIGEST]; 3965 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3966 req_ctx->cbk = ahash_done; 3967 req_ctx->ctx = &req->base; 3968 req_ctx->edesc = edesc; 3969 3970 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3971 if (ret == -EINPROGRESS || 3972 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3973 return ret; 3974 3975unmap: 3976 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3977 qi_cache_free(edesc); 3978 return ret; 3979} 3980 3981static int ahash_update_no_ctx(struct ahash_request *req) 3982{ 3983 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3984 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3985 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3986 struct caam_request *req_ctx = &state->caam_req; 3987 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3988 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3989 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3990 GFP_KERNEL : GFP_ATOMIC; 3991 u8 *buf = state->buf; 3992 int *buflen = &state->buflen; 3993 int *next_buflen = &state->next_buflen; 3994 int in_len = *buflen + req->nbytes, to_hash; 3995 int qm_sg_bytes, src_nents, mapped_nents; 3996 struct ahash_edesc *edesc; 3997 int ret = 0; 3998 3999 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 4000 to_hash = in_len - *next_buflen; 4001 4002 if (to_hash) { 4003 struct dpaa2_sg_entry *sg_table; 4004 int src_len = req->nbytes - *next_buflen; 4005 4006 src_nents = sg_nents_for_len(req->src, src_len); 4007 if (src_nents < 0) { 4008 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4009 return src_nents; 4010 } 4011 4012 if (src_nents) { 4013 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4014 DMA_TO_DEVICE); 4015 if (!mapped_nents) { 4016 dev_err(ctx->dev, "unable to DMA map source\n"); 4017 return -ENOMEM; 4018 } 4019 } else { 4020 mapped_nents = 0; 4021 } 4022 4023 /* allocate space for base edesc and link tables */ 4024 edesc = qi_cache_zalloc(flags); 4025 if (!edesc) { 4026 dma_unmap_sg(ctx->dev, req->src, src_nents, 4027 DMA_TO_DEVICE); 4028 return -ENOMEM; 4029 } 4030 4031 edesc->src_nents = src_nents; 4032 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 4033 sizeof(*sg_table); 4034 sg_table = &edesc->sgt[0]; 4035 4036 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4037 if (ret) 4038 goto unmap_ctx; 4039 4040 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 4041 4042 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4043 qm_sg_bytes, DMA_TO_DEVICE); 4044 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4045 dev_err(ctx->dev, "unable to map S/G table\n"); 4046 ret = -ENOMEM; 4047 goto unmap_ctx; 4048 } 4049 edesc->qm_sg_bytes = qm_sg_bytes; 4050 4051 state->ctx_dma_len = ctx->ctx_len; 4052 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4053 ctx->ctx_len, DMA_FROM_DEVICE); 4054 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4055 dev_err(ctx->dev, "unable to map ctx\n"); 4056 state->ctx_dma = 0; 4057 ret = -ENOMEM; 4058 goto unmap_ctx; 4059 } 4060 4061 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4062 dpaa2_fl_set_final(in_fle, true); 4063 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4064 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4065 dpaa2_fl_set_len(in_fle, to_hash); 4066 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4067 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4068 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4069 4070 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4071 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4072 req_ctx->cbk = ahash_done_ctx_dst; 4073 req_ctx->ctx = &req->base; 4074 req_ctx->edesc = edesc; 4075 4076 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4077 if (ret != -EINPROGRESS && 4078 !(ret == -EBUSY && 4079 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4080 goto unmap_ctx; 4081 4082 state->update = ahash_update_ctx; 4083 state->finup = ahash_finup_ctx; 4084 state->final = ahash_final_ctx; 4085 } else if (*next_buflen) { 4086 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4087 req->nbytes, 0); 4088 *buflen = *next_buflen; 4089 4090 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4091 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4092 *buflen, 1); 4093 } 4094 4095 return ret; 4096unmap_ctx: 4097 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4098 qi_cache_free(edesc); 4099 return ret; 4100} 4101 4102static int ahash_finup_no_ctx(struct ahash_request *req) 4103{ 4104 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4105 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4106 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4107 struct caam_request *req_ctx = &state->caam_req; 4108 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4109 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4110 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4111 GFP_KERNEL : GFP_ATOMIC; 4112 int buflen = state->buflen; 4113 int qm_sg_bytes, src_nents, mapped_nents; 4114 int digestsize = crypto_ahash_digestsize(ahash); 4115 struct ahash_edesc *edesc; 4116 struct dpaa2_sg_entry *sg_table; 4117 int ret = -ENOMEM; 4118 4119 src_nents = sg_nents_for_len(req->src, req->nbytes); 4120 if (src_nents < 0) { 4121 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4122 return src_nents; 4123 } 4124 4125 if (src_nents) { 4126 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4127 DMA_TO_DEVICE); 4128 if (!mapped_nents) { 4129 dev_err(ctx->dev, "unable to DMA map source\n"); 4130 return ret; 4131 } 4132 } else { 4133 mapped_nents = 0; 4134 } 4135 4136 /* allocate space for base edesc and link tables */ 4137 edesc = qi_cache_zalloc(flags); 4138 if (!edesc) { 4139 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4140 return ret; 4141 } 4142 4143 edesc->src_nents = src_nents; 4144 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4145 sg_table = &edesc->sgt[0]; 4146 4147 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4148 if (ret) 4149 goto unmap; 4150 4151 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4152 4153 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4154 DMA_TO_DEVICE); 4155 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4156 dev_err(ctx->dev, "unable to map S/G table\n"); 4157 ret = -ENOMEM; 4158 goto unmap; 4159 } 4160 edesc->qm_sg_bytes = qm_sg_bytes; 4161 4162 state->ctx_dma_len = digestsize; 4163 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4164 DMA_FROM_DEVICE); 4165 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4166 dev_err(ctx->dev, "unable to map ctx\n"); 4167 state->ctx_dma = 0; 4168 ret = -ENOMEM; 4169 goto unmap; 4170 } 4171 4172 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4173 dpaa2_fl_set_final(in_fle, true); 4174 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4175 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4176 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4177 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4178 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4179 dpaa2_fl_set_len(out_fle, digestsize); 4180 4181 req_ctx->flc = &ctx->flc[DIGEST]; 4182 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4183 req_ctx->cbk = ahash_done; 4184 req_ctx->ctx = &req->base; 4185 req_ctx->edesc = edesc; 4186 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4187 if (ret != -EINPROGRESS && 4188 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4189 goto unmap; 4190 4191 return ret; 4192unmap: 4193 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4194 qi_cache_free(edesc); 4195 return ret; 4196} 4197 4198static int ahash_update_first(struct ahash_request *req) 4199{ 4200 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4201 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4202 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4203 struct caam_request *req_ctx = &state->caam_req; 4204 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4205 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4206 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4207 GFP_KERNEL : GFP_ATOMIC; 4208 u8 *buf = state->buf; 4209 int *buflen = &state->buflen; 4210 int *next_buflen = &state->next_buflen; 4211 int to_hash; 4212 int src_nents, mapped_nents; 4213 struct ahash_edesc *edesc; 4214 int ret = 0; 4215 4216 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4217 1); 4218 to_hash = req->nbytes - *next_buflen; 4219 4220 if (to_hash) { 4221 struct dpaa2_sg_entry *sg_table; 4222 int src_len = req->nbytes - *next_buflen; 4223 4224 src_nents = sg_nents_for_len(req->src, src_len); 4225 if (src_nents < 0) { 4226 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4227 return src_nents; 4228 } 4229 4230 if (src_nents) { 4231 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4232 DMA_TO_DEVICE); 4233 if (!mapped_nents) { 4234 dev_err(ctx->dev, "unable to map source for DMA\n"); 4235 return -ENOMEM; 4236 } 4237 } else { 4238 mapped_nents = 0; 4239 } 4240 4241 /* allocate space for base edesc and link tables */ 4242 edesc = qi_cache_zalloc(flags); 4243 if (!edesc) { 4244 dma_unmap_sg(ctx->dev, req->src, src_nents, 4245 DMA_TO_DEVICE); 4246 return -ENOMEM; 4247 } 4248 4249 edesc->src_nents = src_nents; 4250 sg_table = &edesc->sgt[0]; 4251 4252 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4253 dpaa2_fl_set_final(in_fle, true); 4254 dpaa2_fl_set_len(in_fle, to_hash); 4255 4256 if (mapped_nents > 1) { 4257 int qm_sg_bytes; 4258 4259 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4260 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4261 sizeof(*sg_table); 4262 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4263 qm_sg_bytes, 4264 DMA_TO_DEVICE); 4265 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4266 dev_err(ctx->dev, "unable to map S/G table\n"); 4267 ret = -ENOMEM; 4268 goto unmap_ctx; 4269 } 4270 edesc->qm_sg_bytes = qm_sg_bytes; 4271 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4272 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4273 } else { 4274 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4275 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4276 } 4277 4278 state->ctx_dma_len = ctx->ctx_len; 4279 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4280 ctx->ctx_len, DMA_FROM_DEVICE); 4281 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4282 dev_err(ctx->dev, "unable to map ctx\n"); 4283 state->ctx_dma = 0; 4284 ret = -ENOMEM; 4285 goto unmap_ctx; 4286 } 4287 4288 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4289 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4290 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4291 4292 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4293 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4294 req_ctx->cbk = ahash_done_ctx_dst; 4295 req_ctx->ctx = &req->base; 4296 req_ctx->edesc = edesc; 4297 4298 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4299 if (ret != -EINPROGRESS && 4300 !(ret == -EBUSY && req->base.flags & 4301 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4302 goto unmap_ctx; 4303 4304 state->update = ahash_update_ctx; 4305 state->finup = ahash_finup_ctx; 4306 state->final = ahash_final_ctx; 4307 } else if (*next_buflen) { 4308 state->update = ahash_update_no_ctx; 4309 state->finup = ahash_finup_no_ctx; 4310 state->final = ahash_final_no_ctx; 4311 scatterwalk_map_and_copy(buf, req->src, 0, 4312 req->nbytes, 0); 4313 *buflen = *next_buflen; 4314 4315 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4316 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4317 *buflen, 1); 4318 } 4319 4320 return ret; 4321unmap_ctx: 4322 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4323 qi_cache_free(edesc); 4324 return ret; 4325} 4326 4327static int ahash_finup_first(struct ahash_request *req) 4328{ 4329 return ahash_digest(req); 4330} 4331 4332static int ahash_init(struct ahash_request *req) 4333{ 4334 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4335 4336 state->update = ahash_update_first; 4337 state->finup = ahash_finup_first; 4338 state->final = ahash_final_no_ctx; 4339 4340 state->ctx_dma = 0; 4341 state->ctx_dma_len = 0; 4342 state->buf_dma = 0; 4343 state->buflen = 0; 4344 state->next_buflen = 0; 4345 4346 return 0; 4347} 4348 4349static int ahash_update(struct ahash_request *req) 4350{ 4351 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4352 4353 return state->update(req); 4354} 4355 4356static int ahash_finup(struct ahash_request *req) 4357{ 4358 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4359 4360 return state->finup(req); 4361} 4362 4363static int ahash_final(struct ahash_request *req) 4364{ 4365 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4366 4367 return state->final(req); 4368} 4369 4370static int ahash_export(struct ahash_request *req, void *out) 4371{ 4372 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4373 struct caam_export_state *export = out; 4374 u8 *buf = state->buf; 4375 int len = state->buflen; 4376 4377 memcpy(export->buf, buf, len); 4378 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4379 export->buflen = len; 4380 export->update = state->update; 4381 export->final = state->final; 4382 export->finup = state->finup; 4383 4384 return 0; 4385} 4386 4387static int ahash_import(struct ahash_request *req, const void *in) 4388{ 4389 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4390 const struct caam_export_state *export = in; 4391 4392 memset(state, 0, sizeof(*state)); 4393 memcpy(state->buf, export->buf, export->buflen); 4394 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4395 state->buflen = export->buflen; 4396 state->update = export->update; 4397 state->final = export->final; 4398 state->finup = export->finup; 4399 4400 return 0; 4401} 4402 4403struct caam_hash_template { 4404 char name[CRYPTO_MAX_ALG_NAME]; 4405 char driver_name[CRYPTO_MAX_ALG_NAME]; 4406 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4407 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4408 unsigned int blocksize; 4409 struct ahash_alg template_ahash; 4410 u32 alg_type; 4411}; 4412 4413/* ahash descriptors */ 4414static struct caam_hash_template driver_hash[] = { 4415 { 4416 .name = "sha1", 4417 .driver_name = "sha1-caam-qi2", 4418 .hmac_name = "hmac(sha1)", 4419 .hmac_driver_name = "hmac-sha1-caam-qi2", 4420 .blocksize = SHA1_BLOCK_SIZE, 4421 .template_ahash = { 4422 .init = ahash_init, 4423 .update = ahash_update, 4424 .final = ahash_final, 4425 .finup = ahash_finup, 4426 .digest = ahash_digest, 4427 .export = ahash_export, 4428 .import = ahash_import, 4429 .setkey = ahash_setkey, 4430 .halg = { 4431 .digestsize = SHA1_DIGEST_SIZE, 4432 .statesize = sizeof(struct caam_export_state), 4433 }, 4434 }, 4435 .alg_type = OP_ALG_ALGSEL_SHA1, 4436 }, { 4437 .name = "sha224", 4438 .driver_name = "sha224-caam-qi2", 4439 .hmac_name = "hmac(sha224)", 4440 .hmac_driver_name = "hmac-sha224-caam-qi2", 4441 .blocksize = SHA224_BLOCK_SIZE, 4442 .template_ahash = { 4443 .init = ahash_init, 4444 .update = ahash_update, 4445 .final = ahash_final, 4446 .finup = ahash_finup, 4447 .digest = ahash_digest, 4448 .export = ahash_export, 4449 .import = ahash_import, 4450 .setkey = ahash_setkey, 4451 .halg = { 4452 .digestsize = SHA224_DIGEST_SIZE, 4453 .statesize = sizeof(struct caam_export_state), 4454 }, 4455 }, 4456 .alg_type = OP_ALG_ALGSEL_SHA224, 4457 }, { 4458 .name = "sha256", 4459 .driver_name = "sha256-caam-qi2", 4460 .hmac_name = "hmac(sha256)", 4461 .hmac_driver_name = "hmac-sha256-caam-qi2", 4462 .blocksize = SHA256_BLOCK_SIZE, 4463 .template_ahash = { 4464 .init = ahash_init, 4465 .update = ahash_update, 4466 .final = ahash_final, 4467 .finup = ahash_finup, 4468 .digest = ahash_digest, 4469 .export = ahash_export, 4470 .import = ahash_import, 4471 .setkey = ahash_setkey, 4472 .halg = { 4473 .digestsize = SHA256_DIGEST_SIZE, 4474 .statesize = sizeof(struct caam_export_state), 4475 }, 4476 }, 4477 .alg_type = OP_ALG_ALGSEL_SHA256, 4478 }, { 4479 .name = "sha384", 4480 .driver_name = "sha384-caam-qi2", 4481 .hmac_name = "hmac(sha384)", 4482 .hmac_driver_name = "hmac-sha384-caam-qi2", 4483 .blocksize = SHA384_BLOCK_SIZE, 4484 .template_ahash = { 4485 .init = ahash_init, 4486 .update = ahash_update, 4487 .final = ahash_final, 4488 .finup = ahash_finup, 4489 .digest = ahash_digest, 4490 .export = ahash_export, 4491 .import = ahash_import, 4492 .setkey = ahash_setkey, 4493 .halg = { 4494 .digestsize = SHA384_DIGEST_SIZE, 4495 .statesize = sizeof(struct caam_export_state), 4496 }, 4497 }, 4498 .alg_type = OP_ALG_ALGSEL_SHA384, 4499 }, { 4500 .name = "sha512", 4501 .driver_name = "sha512-caam-qi2", 4502 .hmac_name = "hmac(sha512)", 4503 .hmac_driver_name = "hmac-sha512-caam-qi2", 4504 .blocksize = SHA512_BLOCK_SIZE, 4505 .template_ahash = { 4506 .init = ahash_init, 4507 .update = ahash_update, 4508 .final = ahash_final, 4509 .finup = ahash_finup, 4510 .digest = ahash_digest, 4511 .export = ahash_export, 4512 .import = ahash_import, 4513 .setkey = ahash_setkey, 4514 .halg = { 4515 .digestsize = SHA512_DIGEST_SIZE, 4516 .statesize = sizeof(struct caam_export_state), 4517 }, 4518 }, 4519 .alg_type = OP_ALG_ALGSEL_SHA512, 4520 }, { 4521 .name = "md5", 4522 .driver_name = "md5-caam-qi2", 4523 .hmac_name = "hmac(md5)", 4524 .hmac_driver_name = "hmac-md5-caam-qi2", 4525 .blocksize = MD5_BLOCK_WORDS * 4, 4526 .template_ahash = { 4527 .init = ahash_init, 4528 .update = ahash_update, 4529 .final = ahash_final, 4530 .finup = ahash_finup, 4531 .digest = ahash_digest, 4532 .export = ahash_export, 4533 .import = ahash_import, 4534 .setkey = ahash_setkey, 4535 .halg = { 4536 .digestsize = MD5_DIGEST_SIZE, 4537 .statesize = sizeof(struct caam_export_state), 4538 }, 4539 }, 4540 .alg_type = OP_ALG_ALGSEL_MD5, 4541 } 4542}; 4543 4544struct caam_hash_alg { 4545 struct list_head entry; 4546 struct device *dev; 4547 int alg_type; 4548 bool is_hmac; 4549 struct ahash_alg ahash_alg; 4550}; 4551 4552static int caam_hash_cra_init(struct crypto_tfm *tfm) 4553{ 4554 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4555 struct crypto_alg *base = tfm->__crt_alg; 4556 struct hash_alg_common *halg = 4557 container_of(base, struct hash_alg_common, base); 4558 struct ahash_alg *alg = 4559 container_of(halg, struct ahash_alg, halg); 4560 struct caam_hash_alg *caam_hash = 4561 container_of(alg, struct caam_hash_alg, ahash_alg); 4562 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4563 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4564 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4565 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4566 HASH_MSG_LEN + 32, 4567 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4568 HASH_MSG_LEN + 64, 4569 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4570 dma_addr_t dma_addr; 4571 int i; 4572 4573 ctx->dev = caam_hash->dev; 4574 4575 if (caam_hash->is_hmac) { 4576 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4577 ARRAY_SIZE(ctx->key), 4578 DMA_TO_DEVICE, 4579 DMA_ATTR_SKIP_CPU_SYNC); 4580 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4581 dev_err(ctx->dev, "unable to map key\n"); 4582 return -ENOMEM; 4583 } 4584 } 4585 4586 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4587 DMA_BIDIRECTIONAL, 4588 DMA_ATTR_SKIP_CPU_SYNC); 4589 if (dma_mapping_error(ctx->dev, dma_addr)) { 4590 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4591 if (ctx->adata.key_dma) 4592 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4593 ARRAY_SIZE(ctx->key), 4594 DMA_TO_DEVICE, 4595 DMA_ATTR_SKIP_CPU_SYNC); 4596 return -ENOMEM; 4597 } 4598 4599 for (i = 0; i < HASH_NUM_OP; i++) 4600 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4601 4602 /* copy descriptor header template value */ 4603 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4604 4605 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4606 OP_ALG_ALGSEL_SUBMASK) >> 4607 OP_ALG_ALGSEL_SHIFT]; 4608 4609 crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state)); 4610 4611 /* 4612 * For keyed hash algorithms shared descriptors 4613 * will be created later in setkey() callback 4614 */ 4615 return caam_hash->is_hmac ? 0 : ahash_set_sh_desc(ahash); 4616} 4617 4618static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4619{ 4620 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4621 4622 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4623 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4624 if (ctx->adata.key_dma) 4625 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4626 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4627 DMA_ATTR_SKIP_CPU_SYNC); 4628} 4629 4630static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4631 struct caam_hash_template *template, bool keyed) 4632{ 4633 struct caam_hash_alg *t_alg; 4634 struct ahash_alg *halg; 4635 struct crypto_alg *alg; 4636 4637 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4638 if (!t_alg) 4639 return ERR_PTR(-ENOMEM); 4640 4641 t_alg->ahash_alg = template->template_ahash; 4642 halg = &t_alg->ahash_alg; 4643 alg = &halg->halg.base; 4644 4645 if (keyed) { 4646 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4647 template->hmac_name); 4648 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4649 template->hmac_driver_name); 4650 t_alg->is_hmac = true; 4651 } else { 4652 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4653 template->name); 4654 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4655 template->driver_name); 4656 t_alg->ahash_alg.setkey = NULL; 4657 t_alg->is_hmac = false; 4658 } 4659 alg->cra_module = THIS_MODULE; 4660 alg->cra_init = caam_hash_cra_init; 4661 alg->cra_exit = caam_hash_cra_exit; 4662 alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding(); 4663 alg->cra_priority = CAAM_CRA_PRIORITY; 4664 alg->cra_blocksize = template->blocksize; 4665 alg->cra_alignmask = 0; 4666 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4667 4668 t_alg->alg_type = template->alg_type; 4669 t_alg->dev = dev; 4670 4671 return t_alg; 4672} 4673 4674static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4675{ 4676 struct dpaa2_caam_priv_per_cpu *ppriv; 4677 4678 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4679 napi_schedule_irqoff(&ppriv->napi); 4680} 4681 4682static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4683{ 4684 struct device *dev = priv->dev; 4685 struct dpaa2_io_notification_ctx *nctx; 4686 struct dpaa2_caam_priv_per_cpu *ppriv; 4687 int err, i = 0, cpu; 4688 4689 for_each_online_cpu(cpu) { 4690 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4691 ppriv->priv = priv; 4692 nctx = &ppriv->nctx; 4693 nctx->is_cdan = 0; 4694 nctx->id = ppriv->rsp_fqid; 4695 nctx->desired_cpu = cpu; 4696 nctx->cb = dpaa2_caam_fqdan_cb; 4697 4698 /* Register notification callbacks */ 4699 ppriv->dpio = dpaa2_io_service_select(cpu); 4700 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4701 if (unlikely(err)) { 4702 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4703 nctx->cb = NULL; 4704 /* 4705 * If no affine DPIO for this core, there's probably 4706 * none available for next cores either. Signal we want 4707 * to retry later, in case the DPIO devices weren't 4708 * probed yet. 4709 */ 4710 err = -EPROBE_DEFER; 4711 goto err; 4712 } 4713 4714 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4715 dev); 4716 if (unlikely(!ppriv->store)) { 4717 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4718 err = -ENOMEM; 4719 goto err; 4720 } 4721 4722 if (++i == priv->num_pairs) 4723 break; 4724 } 4725 4726 return 0; 4727 4728err: 4729 for_each_online_cpu(cpu) { 4730 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4731 if (!ppriv->nctx.cb) 4732 break; 4733 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4734 } 4735 4736 for_each_online_cpu(cpu) { 4737 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4738 if (!ppriv->store) 4739 break; 4740 dpaa2_io_store_destroy(ppriv->store); 4741 } 4742 4743 return err; 4744} 4745 4746static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4747{ 4748 struct dpaa2_caam_priv_per_cpu *ppriv; 4749 int i = 0, cpu; 4750 4751 for_each_online_cpu(cpu) { 4752 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4753 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4754 priv->dev); 4755 dpaa2_io_store_destroy(ppriv->store); 4756 4757 if (++i == priv->num_pairs) 4758 return; 4759 } 4760} 4761 4762static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4763{ 4764 struct dpseci_rx_queue_cfg rx_queue_cfg; 4765 struct device *dev = priv->dev; 4766 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4767 struct dpaa2_caam_priv_per_cpu *ppriv; 4768 int err = 0, i = 0, cpu; 4769 4770 /* Configure Rx queues */ 4771 for_each_online_cpu(cpu) { 4772 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4773 4774 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4775 DPSECI_QUEUE_OPT_USER_CTX; 4776 rx_queue_cfg.order_preservation_en = 0; 4777 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4778 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4779 /* 4780 * Rx priority (WQ) doesn't really matter, since we use 4781 * pull mode, i.e. volatile dequeues from specific FQs 4782 */ 4783 rx_queue_cfg.dest_cfg.priority = 0; 4784 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4785 4786 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4787 &rx_queue_cfg); 4788 if (err) { 4789 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4790 err); 4791 return err; 4792 } 4793 4794 if (++i == priv->num_pairs) 4795 break; 4796 } 4797 4798 return err; 4799} 4800 4801static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4802{ 4803 struct device *dev = priv->dev; 4804 4805 if (!priv->cscn_mem) 4806 return; 4807 4808 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4809 kfree(priv->cscn_mem); 4810} 4811 4812static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4813{ 4814 struct device *dev = priv->dev; 4815 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4816 int err; 4817 4818 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4819 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4820 if (err) 4821 dev_err(dev, "dpseci_reset() failed\n"); 4822 } 4823 4824 dpaa2_dpseci_congestion_free(priv); 4825 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4826} 4827 4828static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4829 const struct dpaa2_fd *fd) 4830{ 4831 struct caam_request *req; 4832 u32 fd_err; 4833 4834 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4835 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4836 return; 4837 } 4838 4839 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4840 if (unlikely(fd_err)) 4841 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4842 4843 /* 4844 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4845 * in FD[ERR] or FD[FRC]. 4846 */ 4847 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4848 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4849 DMA_BIDIRECTIONAL); 4850 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4851} 4852 4853static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4854{ 4855 int err; 4856 4857 /* Retry while portal is busy */ 4858 do { 4859 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4860 ppriv->store); 4861 } while (err == -EBUSY); 4862 4863 if (unlikely(err)) 4864 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4865 4866 return err; 4867} 4868 4869static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4870{ 4871 struct dpaa2_dq *dq; 4872 int cleaned = 0, is_last; 4873 4874 do { 4875 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4876 if (unlikely(!dq)) { 4877 if (unlikely(!is_last)) { 4878 dev_dbg(ppriv->priv->dev, 4879 "FQ %d returned no valid frames\n", 4880 ppriv->rsp_fqid); 4881 /* 4882 * MUST retry until we get some sort of 4883 * valid response token (be it "empty dequeue" 4884 * or a valid frame). 4885 */ 4886 continue; 4887 } 4888 break; 4889 } 4890 4891 /* Process FD */ 4892 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4893 cleaned++; 4894 } while (!is_last); 4895 4896 return cleaned; 4897} 4898 4899static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4900{ 4901 struct dpaa2_caam_priv_per_cpu *ppriv; 4902 struct dpaa2_caam_priv *priv; 4903 int err, cleaned = 0, store_cleaned; 4904 4905 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4906 priv = ppriv->priv; 4907 4908 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4909 return 0; 4910 4911 do { 4912 store_cleaned = dpaa2_caam_store_consume(ppriv); 4913 cleaned += store_cleaned; 4914 4915 if (store_cleaned == 0 || 4916 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4917 break; 4918 4919 /* Try to dequeue some more */ 4920 err = dpaa2_caam_pull_fq(ppriv); 4921 if (unlikely(err)) 4922 break; 4923 } while (1); 4924 4925 if (cleaned < budget) { 4926 napi_complete_done(napi, cleaned); 4927 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4928 if (unlikely(err)) 4929 dev_err(priv->dev, "Notification rearm failed: %d\n", 4930 err); 4931 } 4932 4933 return cleaned; 4934} 4935 4936static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4937 u16 token) 4938{ 4939 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4940 struct device *dev = priv->dev; 4941 unsigned int alignmask; 4942 int err; 4943 4944 /* 4945 * Congestion group feature supported starting with DPSECI API v5.1 4946 * and only when object has been created with this capability. 4947 */ 4948 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4949 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4950 return 0; 4951 4952 alignmask = DPAA2_CSCN_ALIGN - 1; 4953 alignmask |= dma_get_cache_alignment() - 1; 4954 priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1), 4955 GFP_KERNEL); 4956 if (!priv->cscn_mem) 4957 return -ENOMEM; 4958 4959 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem, 4960 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4961 if (dma_mapping_error(dev, priv->cscn_dma)) { 4962 dev_err(dev, "Error mapping CSCN memory area\n"); 4963 err = -ENOMEM; 4964 goto err_dma_map; 4965 } 4966 4967 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4968 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4969 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4970 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4971 cong_notif_cfg.message_iova = priv->cscn_dma; 4972 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4973 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4974 DPSECI_CGN_MODE_COHERENT_WRITE; 4975 4976 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4977 &cong_notif_cfg); 4978 if (err) { 4979 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4980 goto err_set_cong; 4981 } 4982 4983 return 0; 4984 4985err_set_cong: 4986 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4987err_dma_map: 4988 kfree(priv->cscn_mem); 4989 4990 return err; 4991} 4992 4993static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4994{ 4995 struct device *dev = &ls_dev->dev; 4996 struct dpaa2_caam_priv *priv; 4997 struct dpaa2_caam_priv_per_cpu *ppriv; 4998 int err, cpu; 4999 u8 i; 5000 5001 priv = dev_get_drvdata(dev); 5002 5003 priv->dev = dev; 5004 priv->dpsec_id = ls_dev->obj_desc.id; 5005 5006 /* Get a handle for the DPSECI this interface is associate with */ 5007 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 5008 if (err) { 5009 dev_err(dev, "dpseci_open() failed: %d\n", err); 5010 goto err_open; 5011 } 5012 5013 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 5014 &priv->minor_ver); 5015 if (err) { 5016 dev_err(dev, "dpseci_get_api_version() failed\n"); 5017 goto err_get_vers; 5018 } 5019 5020 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 5021 5022 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 5023 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 5024 if (err) { 5025 dev_err(dev, "dpseci_reset() failed\n"); 5026 goto err_get_vers; 5027 } 5028 } 5029 5030 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 5031 &priv->dpseci_attr); 5032 if (err) { 5033 dev_err(dev, "dpseci_get_attributes() failed\n"); 5034 goto err_get_vers; 5035 } 5036 5037 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 5038 &priv->sec_attr); 5039 if (err) { 5040 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 5041 goto err_get_vers; 5042 } 5043 5044 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 5045 if (err) { 5046 dev_err(dev, "setup_congestion() failed\n"); 5047 goto err_get_vers; 5048 } 5049 5050 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 5051 priv->dpseci_attr.num_tx_queues); 5052 if (priv->num_pairs > num_online_cpus()) { 5053 dev_warn(dev, "%d queues won't be used\n", 5054 priv->num_pairs - num_online_cpus()); 5055 priv->num_pairs = num_online_cpus(); 5056 } 5057 5058 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 5059 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5060 &priv->rx_queue_attr[i]); 5061 if (err) { 5062 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 5063 goto err_get_rx_queue; 5064 } 5065 } 5066 5067 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 5068 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5069 &priv->tx_queue_attr[i]); 5070 if (err) { 5071 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 5072 goto err_get_rx_queue; 5073 } 5074 } 5075 5076 i = 0; 5077 for_each_online_cpu(cpu) { 5078 u8 j; 5079 5080 j = i % priv->num_pairs; 5081 5082 ppriv = per_cpu_ptr(priv->ppriv, cpu); 5083 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 5084 5085 /* 5086 * Allow all cores to enqueue, while only some of them 5087 * will take part in dequeuing. 5088 */ 5089 if (++i > priv->num_pairs) 5090 continue; 5091 5092 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 5093 ppriv->prio = j; 5094 5095 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 5096 priv->rx_queue_attr[j].fqid, 5097 priv->tx_queue_attr[j].fqid); 5098 5099 ppriv->net_dev.dev = *dev; 5100 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 5101 netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi, 5102 dpaa2_dpseci_poll, 5103 DPAA2_CAAM_NAPI_WEIGHT); 5104 } 5105 5106 return 0; 5107 5108err_get_rx_queue: 5109 dpaa2_dpseci_congestion_free(priv); 5110err_get_vers: 5111 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5112err_open: 5113 return err; 5114} 5115 5116static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5117{ 5118 struct device *dev = priv->dev; 5119 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5120 struct dpaa2_caam_priv_per_cpu *ppriv; 5121 int i; 5122 5123 for (i = 0; i < priv->num_pairs; i++) { 5124 ppriv = per_cpu_ptr(priv->ppriv, i); 5125 napi_enable(&ppriv->napi); 5126 } 5127 5128 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5129} 5130 5131static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5132{ 5133 struct device *dev = priv->dev; 5134 struct dpaa2_caam_priv_per_cpu *ppriv; 5135 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5136 int i, err = 0, enabled; 5137 5138 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5139 if (err) { 5140 dev_err(dev, "dpseci_disable() failed\n"); 5141 return err; 5142 } 5143 5144 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5145 if (err) { 5146 dev_err(dev, "dpseci_is_enabled() failed\n"); 5147 return err; 5148 } 5149 5150 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 5151 5152 for (i = 0; i < priv->num_pairs; i++) { 5153 ppriv = per_cpu_ptr(priv->ppriv, i); 5154 napi_disable(&ppriv->napi); 5155 netif_napi_del(&ppriv->napi); 5156 } 5157 5158 return 0; 5159} 5160 5161static struct list_head hash_list; 5162 5163static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5164{ 5165 struct device *dev; 5166 struct dpaa2_caam_priv *priv; 5167 int i, err = 0; 5168 bool registered = false; 5169 5170 /* 5171 * There is no way to get CAAM endianness - there is no direct register 5172 * space access and MC f/w does not provide this attribute. 5173 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5174 * property. 5175 */ 5176 caam_little_end = true; 5177 5178 caam_imx = false; 5179 5180 dev = &dpseci_dev->dev; 5181 5182 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5183 if (!priv) 5184 return -ENOMEM; 5185 5186 dev_set_drvdata(dev, priv); 5187 5188 priv->domain = iommu_get_domain_for_dev(dev); 5189 5190 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5191 0, 0, NULL); 5192 if (!qi_cache) { 5193 dev_err(dev, "Can't allocate SEC cache\n"); 5194 return -ENOMEM; 5195 } 5196 5197 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5198 if (err) { 5199 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5200 goto err_dma_mask; 5201 } 5202 5203 /* Obtain a MC portal */ 5204 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5205 if (err) { 5206 if (err == -ENXIO) 5207 err = -EPROBE_DEFER; 5208 else 5209 dev_err(dev, "MC portal allocation failed\n"); 5210 5211 goto err_dma_mask; 5212 } 5213 5214 priv->ppriv = alloc_percpu(*priv->ppriv); 5215 if (!priv->ppriv) { 5216 dev_err(dev, "alloc_percpu() failed\n"); 5217 err = -ENOMEM; 5218 goto err_alloc_ppriv; 5219 } 5220 5221 /* DPSECI initialization */ 5222 err = dpaa2_dpseci_setup(dpseci_dev); 5223 if (err) { 5224 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5225 goto err_dpseci_setup; 5226 } 5227 5228 /* DPIO */ 5229 err = dpaa2_dpseci_dpio_setup(priv); 5230 if (err) { 5231 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); 5232 goto err_dpio_setup; 5233 } 5234 5235 /* DPSECI binding to DPIO */ 5236 err = dpaa2_dpseci_bind(priv); 5237 if (err) { 5238 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5239 goto err_bind; 5240 } 5241 5242 /* DPSECI enable */ 5243 err = dpaa2_dpseci_enable(priv); 5244 if (err) { 5245 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5246 goto err_bind; 5247 } 5248 5249 dpaa2_dpseci_debugfs_init(priv); 5250 5251 /* register crypto algorithms the device supports */ 5252 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5253 struct caam_skcipher_alg *t_alg = driver_algs + i; 5254 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5255 5256 /* Skip DES algorithms if not supported by device */ 5257 if (!priv->sec_attr.des_acc_num && 5258 (alg_sel == OP_ALG_ALGSEL_3DES || 5259 alg_sel == OP_ALG_ALGSEL_DES)) 5260 continue; 5261 5262 /* Skip AES algorithms if not supported by device */ 5263 if (!priv->sec_attr.aes_acc_num && 5264 alg_sel == OP_ALG_ALGSEL_AES) 5265 continue; 5266 5267 /* Skip CHACHA20 algorithms if not supported by device */ 5268 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5269 !priv->sec_attr.ccha_acc_num) 5270 continue; 5271 5272 t_alg->caam.dev = dev; 5273 caam_skcipher_alg_init(t_alg); 5274 5275 err = crypto_register_skcipher(&t_alg->skcipher); 5276 if (err) { 5277 dev_warn(dev, "%s alg registration failed: %d\n", 5278 t_alg->skcipher.base.cra_driver_name, err); 5279 continue; 5280 } 5281 5282 t_alg->registered = true; 5283 registered = true; 5284 } 5285 5286 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5287 struct caam_aead_alg *t_alg = driver_aeads + i; 5288 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5289 OP_ALG_ALGSEL_MASK; 5290 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5291 OP_ALG_ALGSEL_MASK; 5292 5293 /* Skip DES algorithms if not supported by device */ 5294 if (!priv->sec_attr.des_acc_num && 5295 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5296 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5297 continue; 5298 5299 /* Skip AES algorithms if not supported by device */ 5300 if (!priv->sec_attr.aes_acc_num && 5301 c1_alg_sel == OP_ALG_ALGSEL_AES) 5302 continue; 5303 5304 /* Skip CHACHA20 algorithms if not supported by device */ 5305 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5306 !priv->sec_attr.ccha_acc_num) 5307 continue; 5308 5309 /* Skip POLY1305 algorithms if not supported by device */ 5310 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5311 !priv->sec_attr.ptha_acc_num) 5312 continue; 5313 5314 /* 5315 * Skip algorithms requiring message digests 5316 * if MD not supported by device. 5317 */ 5318 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5319 !priv->sec_attr.md_acc_num) 5320 continue; 5321 5322 t_alg->caam.dev = dev; 5323 caam_aead_alg_init(t_alg); 5324 5325 err = crypto_register_aead(&t_alg->aead); 5326 if (err) { 5327 dev_warn(dev, "%s alg registration failed: %d\n", 5328 t_alg->aead.base.cra_driver_name, err); 5329 continue; 5330 } 5331 5332 t_alg->registered = true; 5333 registered = true; 5334 } 5335 if (registered) 5336 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5337 5338 /* register hash algorithms the device supports */ 5339 INIT_LIST_HEAD(&hash_list); 5340 5341 /* 5342 * Skip registration of any hashing algorithms if MD block 5343 * is not present. 5344 */ 5345 if (!priv->sec_attr.md_acc_num) 5346 return 0; 5347 5348 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5349 struct caam_hash_alg *t_alg; 5350 struct caam_hash_template *alg = driver_hash + i; 5351 5352 /* register hmac version */ 5353 t_alg = caam_hash_alloc(dev, alg, true); 5354 if (IS_ERR(t_alg)) { 5355 err = PTR_ERR(t_alg); 5356 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5357 alg->hmac_driver_name, err); 5358 continue; 5359 } 5360 5361 err = crypto_register_ahash(&t_alg->ahash_alg); 5362 if (err) { 5363 dev_warn(dev, "%s alg registration failed: %d\n", 5364 t_alg->ahash_alg.halg.base.cra_driver_name, 5365 err); 5366 kfree(t_alg); 5367 } else { 5368 list_add_tail(&t_alg->entry, &hash_list); 5369 } 5370 5371 /* register unkeyed version */ 5372 t_alg = caam_hash_alloc(dev, alg, false); 5373 if (IS_ERR(t_alg)) { 5374 err = PTR_ERR(t_alg); 5375 dev_warn(dev, "%s alg allocation failed: %d\n", 5376 alg->driver_name, err); 5377 continue; 5378 } 5379 5380 err = crypto_register_ahash(&t_alg->ahash_alg); 5381 if (err) { 5382 dev_warn(dev, "%s alg registration failed: %d\n", 5383 t_alg->ahash_alg.halg.base.cra_driver_name, 5384 err); 5385 kfree(t_alg); 5386 } else { 5387 list_add_tail(&t_alg->entry, &hash_list); 5388 } 5389 } 5390 if (!list_empty(&hash_list)) 5391 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5392 5393 return err; 5394 5395err_bind: 5396 dpaa2_dpseci_dpio_free(priv); 5397err_dpio_setup: 5398 dpaa2_dpseci_free(priv); 5399err_dpseci_setup: 5400 free_percpu(priv->ppriv); 5401err_alloc_ppriv: 5402 fsl_mc_portal_free(priv->mc_io); 5403err_dma_mask: 5404 kmem_cache_destroy(qi_cache); 5405 5406 return err; 5407} 5408 5409static void __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5410{ 5411 struct device *dev; 5412 struct dpaa2_caam_priv *priv; 5413 int i; 5414 5415 dev = &ls_dev->dev; 5416 priv = dev_get_drvdata(dev); 5417 5418 dpaa2_dpseci_debugfs_exit(priv); 5419 5420 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5421 struct caam_aead_alg *t_alg = driver_aeads + i; 5422 5423 if (t_alg->registered) 5424 crypto_unregister_aead(&t_alg->aead); 5425 } 5426 5427 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5428 struct caam_skcipher_alg *t_alg = driver_algs + i; 5429 5430 if (t_alg->registered) 5431 crypto_unregister_skcipher(&t_alg->skcipher); 5432 } 5433 5434 if (hash_list.next) { 5435 struct caam_hash_alg *t_hash_alg, *p; 5436 5437 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5438 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5439 list_del(&t_hash_alg->entry); 5440 kfree(t_hash_alg); 5441 } 5442 } 5443 5444 dpaa2_dpseci_disable(priv); 5445 dpaa2_dpseci_dpio_free(priv); 5446 dpaa2_dpseci_free(priv); 5447 free_percpu(priv->ppriv); 5448 fsl_mc_portal_free(priv->mc_io); 5449 kmem_cache_destroy(qi_cache); 5450} 5451 5452int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5453{ 5454 struct dpaa2_fd fd; 5455 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5456 struct dpaa2_caam_priv_per_cpu *ppriv; 5457 int err = 0, i; 5458 5459 if (IS_ERR(req)) 5460 return PTR_ERR(req); 5461 5462 if (priv->cscn_mem) { 5463 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5464 DPAA2_CSCN_SIZE, 5465 DMA_FROM_DEVICE); 5466 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) { 5467 dev_dbg_ratelimited(dev, "Dropping request\n"); 5468 return -EBUSY; 5469 } 5470 } 5471 5472 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5473 5474 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5475 DMA_BIDIRECTIONAL); 5476 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5477 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5478 goto err_out; 5479 } 5480 5481 memset(&fd, 0, sizeof(fd)); 5482 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5483 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5484 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5485 dpaa2_fd_set_flc(&fd, req->flc_dma); 5486 5487 ppriv = raw_cpu_ptr(priv->ppriv); 5488 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5489 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5490 &fd); 5491 if (err != -EBUSY) 5492 break; 5493 5494 cpu_relax(); 5495 } 5496 5497 if (unlikely(err)) { 5498 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5499 goto err_out; 5500 } 5501 5502 return -EINPROGRESS; 5503 5504err_out: 5505 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5506 DMA_BIDIRECTIONAL); 5507 return -EIO; 5508} 5509EXPORT_SYMBOL(dpaa2_caam_enqueue); 5510 5511static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5512 { 5513 .vendor = FSL_MC_VENDOR_FREESCALE, 5514 .obj_type = "dpseci", 5515 }, 5516 { .vendor = 0x0 } 5517}; 5518MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); 5519 5520static struct fsl_mc_driver dpaa2_caam_driver = { 5521 .driver = { 5522 .name = KBUILD_MODNAME, 5523 .owner = THIS_MODULE, 5524 }, 5525 .probe = dpaa2_caam_probe, 5526 .remove = dpaa2_caam_remove, 5527 .match_id_table = dpaa2_caam_match_id_table 5528}; 5529 5530MODULE_LICENSE("Dual BSD/GPL"); 5531MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5532MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5533 5534module_fsl_mc_driver(dpaa2_caam_driver); 5535