1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (C) 2020 Marvell. */ 3 4#include <crypto/aes.h> 5#include <crypto/authenc.h> 6#include <crypto/cryptd.h> 7#include <crypto/des.h> 8#include <crypto/internal/aead.h> 9#include <crypto/sha1.h> 10#include <crypto/sha2.h> 11#include <crypto/xts.h> 12#include <crypto/gcm.h> 13#include <crypto/scatterwalk.h> 14#include <linux/rtnetlink.h> 15#include <linux/sort.h> 16#include <linux/module.h> 17#include "otx2_cptvf.h" 18#include "otx2_cptvf_algs.h" 19#include "otx2_cpt_reqmgr.h" 20#include "cn10k_cpt.h" 21 22/* Size of salt in AES GCM mode */ 23#define AES_GCM_SALT_SIZE 4 24/* Size of IV in AES GCM mode */ 25#define AES_GCM_IV_SIZE 8 26/* Size of ICV (Integrity Check Value) in AES GCM mode */ 27#define AES_GCM_ICV_SIZE 16 28/* Offset of IV in AES GCM mode */ 29#define AES_GCM_IV_OFFSET 8 30#define CONTROL_WORD_LEN 8 31#define KEY2_OFFSET 48 32#define DMA_MODE_FLAG(dma_mode) \ 33 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0) 34 35/* Truncated SHA digest size */ 36#define SHA1_TRUNC_DIGEST_SIZE 12 37#define SHA256_TRUNC_DIGEST_SIZE 16 38#define SHA384_TRUNC_DIGEST_SIZE 24 39#define SHA512_TRUNC_DIGEST_SIZE 32 40 41static DEFINE_MUTEX(mutex); 42static int is_crypto_registered; 43 44struct cpt_device_desc { 45 struct pci_dev *dev; 46 int num_queues; 47}; 48 49struct cpt_device_table { 50 atomic_t count; 51 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM]; 52}; 53 54static struct cpt_device_table se_devices = { 55 .count = ATOMIC_INIT(0) 56}; 57 58static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) 59{ 60 int count; 61 62 count = atomic_read(&se_devices.count); 63 if (count < 1) 64 return -ENODEV; 65 66 *cpu_num = get_cpu(); 67 /* 68 * On OcteonTX2 platform CPT instruction queue is bound to each 69 * local function LF, in turn LFs can be attached to PF 70 * or VF therefore we always use first device. We get maximum 71 * performance if one CPT queue is available for each cpu 72 * otherwise CPT queues need to be shared between cpus. 73 */ 74 if (*cpu_num >= se_devices.desc[0].num_queues) 75 *cpu_num %= se_devices.desc[0].num_queues; 76 *pdev = se_devices.desc[0].dev; 77 78 put_cpu(); 79 80 return 0; 81} 82 83static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) 84{ 85 struct otx2_cpt_req_ctx *rctx; 86 struct aead_request *req; 87 struct crypto_aead *tfm; 88 89 req = container_of(cpt_req->areq, struct aead_request, base); 90 tfm = crypto_aead_reqtfm(req); 91 rctx = aead_request_ctx_dma(req); 92 if (memcmp(rctx->fctx.hmac.s.hmac_calc, 93 rctx->fctx.hmac.s.hmac_recv, 94 crypto_aead_authsize(tfm)) != 0) 95 return -EBADMSG; 96 97 return 0; 98} 99 100static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2) 101{ 102 struct otx2_cpt_inst_info *inst_info = arg2; 103 struct crypto_async_request *areq = arg1; 104 struct otx2_cpt_req_info *cpt_req; 105 struct pci_dev *pdev; 106 107 if (inst_info) { 108 cpt_req = inst_info->req; 109 if (!status) { 110 /* 111 * When selected cipher is NULL we need to manually 112 * verify whether calculated hmac value matches 113 * received hmac value 114 */ 115 if (cpt_req->req_type == 116 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ && 117 !cpt_req->is_enc) 118 status = validate_hmac_cipher_null(cpt_req); 119 } 120 pdev = inst_info->pdev; 121 otx2_cpt_info_destroy(pdev, inst_info); 122 } 123 if (areq) 124 crypto_request_complete(areq, status); 125} 126 127static void output_iv_copyback(struct crypto_async_request *areq) 128{ 129 struct otx2_cpt_req_info *req_info; 130 struct otx2_cpt_req_ctx *rctx; 131 struct skcipher_request *sreq; 132 struct crypto_skcipher *stfm; 133 struct otx2_cpt_enc_ctx *ctx; 134 u32 start, ivsize; 135 136 sreq = container_of(areq, struct skcipher_request, base); 137 stfm = crypto_skcipher_reqtfm(sreq); 138 ctx = crypto_skcipher_ctx(stfm); 139 if (ctx->cipher_type == OTX2_CPT_AES_CBC || 140 ctx->cipher_type == OTX2_CPT_DES3_CBC) { 141 rctx = skcipher_request_ctx_dma(sreq); 142 req_info = &rctx->cpt_req; 143 ivsize = crypto_skcipher_ivsize(stfm); 144 start = sreq->cryptlen - ivsize; 145 146 if (req_info->is_enc) { 147 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, 148 ivsize, 0); 149 } else { 150 if (sreq->src != sreq->dst) { 151 scatterwalk_map_and_copy(sreq->iv, sreq->src, 152 start, ivsize, 0); 153 } else { 154 memcpy(sreq->iv, req_info->iv_out, ivsize); 155 kfree(req_info->iv_out); 156 } 157 } 158 } 159} 160 161static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2) 162{ 163 struct otx2_cpt_inst_info *inst_info = arg2; 164 struct crypto_async_request *areq = arg1; 165 struct pci_dev *pdev; 166 167 if (areq) { 168 if (!status) 169 output_iv_copyback(areq); 170 if (inst_info) { 171 pdev = inst_info->pdev; 172 otx2_cpt_info_destroy(pdev, inst_info); 173 } 174 crypto_request_complete(areq, status); 175 } 176} 177 178static inline void update_input_data(struct otx2_cpt_req_info *req_info, 179 struct scatterlist *inp_sg, 180 u32 nbytes, u32 *argcnt) 181{ 182 req_info->req.dlen += nbytes; 183 184 while (nbytes) { 185 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length; 186 u8 *ptr = sg_virt(inp_sg); 187 188 req_info->in[*argcnt].vptr = (void *)ptr; 189 req_info->in[*argcnt].size = len; 190 nbytes -= len; 191 ++(*argcnt); 192 inp_sg = sg_next(inp_sg); 193 } 194} 195 196static inline void update_output_data(struct otx2_cpt_req_info *req_info, 197 struct scatterlist *outp_sg, 198 u32 offset, u32 nbytes, u32 *argcnt) 199{ 200 u32 len, sg_len; 201 u8 *ptr; 202 203 req_info->rlen += nbytes; 204 205 while (nbytes) { 206 sg_len = outp_sg->length - offset; 207 len = (nbytes < sg_len) ? nbytes : sg_len; 208 ptr = sg_virt(outp_sg); 209 210 req_info->out[*argcnt].vptr = (void *) (ptr + offset); 211 req_info->out[*argcnt].size = len; 212 nbytes -= len; 213 ++(*argcnt); 214 offset = 0; 215 outp_sg = sg_next(outp_sg); 216 } 217} 218 219static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, 220 u32 *argcnt) 221{ 222 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 223 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 224 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 225 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 226 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; 227 int ivsize = crypto_skcipher_ivsize(stfm); 228 u32 start = req->cryptlen - ivsize; 229 gfp_t flags; 230 231 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 232 GFP_KERNEL : GFP_ATOMIC; 233 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 234 req_info->ctrl.s.se_req = 1; 235 236 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | 237 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 238 if (enc) { 239 req_info->req.opcode.s.minor = 2; 240 } else { 241 req_info->req.opcode.s.minor = 3; 242 if ((ctx->cipher_type == OTX2_CPT_AES_CBC || 243 ctx->cipher_type == OTX2_CPT_DES3_CBC) && 244 req->src == req->dst) { 245 req_info->iv_out = kmalloc(ivsize, flags); 246 if (!req_info->iv_out) 247 return -ENOMEM; 248 249 scatterwalk_map_and_copy(req_info->iv_out, req->src, 250 start, ivsize, 0); 251 } 252 } 253 /* Encryption data length */ 254 req_info->req.param1 = req->cryptlen; 255 /* Authentication data length */ 256 req_info->req.param2 = 0; 257 258 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 259 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 260 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; 261 262 if (ctx->cipher_type == OTX2_CPT_AES_XTS) 263 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 264 else 265 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 266 267 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); 268 269 cpu_to_be64s(&fctx->enc.enc_ctrl.u); 270 271 /* 272 * Storing Packet Data Information in offset 273 * Control Word First 8 bytes 274 */ 275 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 276 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 277 req_info->req.dlen += CONTROL_WORD_LEN; 278 ++(*argcnt); 279 280 req_info->in[*argcnt].vptr = (u8 *)fctx; 281 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); 282 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); 283 284 ++(*argcnt); 285 286 return 0; 287} 288 289static inline int create_input_list(struct skcipher_request *req, u32 enc, 290 u32 enc_iv_len) 291{ 292 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 293 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 294 u32 argcnt = 0; 295 int ret; 296 297 ret = create_ctx_hdr(req, enc, &argcnt); 298 if (ret) 299 return ret; 300 301 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 302 req_info->in_cnt = argcnt; 303 304 return 0; 305} 306 307static inline void create_output_list(struct skcipher_request *req, 308 u32 enc_iv_len) 309{ 310 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 311 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 312 u32 argcnt = 0; 313 314 /* 315 * OUTPUT Buffer Processing 316 * AES encryption/decryption output would be 317 * received in the following format 318 * 319 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 320 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 321 */ 322 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); 323 req_info->out_cnt = argcnt; 324} 325 326static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc) 327{ 328 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 329 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 330 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 331 int ret; 332 333 if (ctx->fbk_cipher) { 334 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher); 335 skcipher_request_set_callback(&rctx->sk_fbk_req, 336 req->base.flags, 337 req->base.complete, 338 req->base.data); 339 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src, 340 req->dst, req->cryptlen, req->iv); 341 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) : 342 crypto_skcipher_decrypt(&rctx->sk_fbk_req); 343 } else { 344 ret = -EINVAL; 345 } 346 return ret; 347} 348 349static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) 350{ 351 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 352 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 353 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 354 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 355 u32 enc_iv_len = crypto_skcipher_ivsize(stfm); 356 struct pci_dev *pdev; 357 int status, cpu_num; 358 359 if (req->cryptlen == 0) 360 return 0; 361 362 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len)) 363 return -EINVAL; 364 365 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE) 366 return skcipher_do_fallback(req, enc); 367 368 /* Clear control words */ 369 rctx->ctrl_word.flags = 0; 370 rctx->fctx.enc.enc_ctrl.u = 0; 371 372 status = create_input_list(req, enc, enc_iv_len); 373 if (status) 374 return status; 375 create_output_list(req, enc_iv_len); 376 377 status = get_se_device(&pdev, &cpu_num); 378 if (status) 379 return status; 380 381 req_info->callback = otx2_cpt_skcipher_callback; 382 req_info->areq = &req->base; 383 req_info->req_type = OTX2_CPT_ENC_DEC_REQ; 384 req_info->is_enc = enc; 385 req_info->is_trunc_hmac = false; 386 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); 387 388 req_info->req.cptr = ctx->er_ctx.hw_ctx; 389 req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; 390 391 /* 392 * We perform an asynchronous send and once 393 * the request is completed the driver would 394 * intimate through registered call back functions 395 */ 396 status = otx2_cpt_do_request(pdev, req_info, cpu_num); 397 398 return status; 399} 400 401static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req) 402{ 403 return cpt_enc_dec(req, true); 404} 405 406static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req) 407{ 408 return cpt_enc_dec(req, false); 409} 410 411static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, 412 const u8 *key, u32 keylen) 413{ 414 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 415 const u8 *key2 = key + (keylen / 2); 416 const u8 *key1 = key; 417 int ret; 418 419 ret = xts_verify_key(tfm, key, keylen); 420 if (ret) 421 return ret; 422 ctx->key_len = keylen; 423 ctx->enc_align_len = 1; 424 memcpy(ctx->enc_key, key1, keylen / 2); 425 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 426 ctx->cipher_type = OTX2_CPT_AES_XTS; 427 switch (ctx->key_len) { 428 case 2 * AES_KEYSIZE_128: 429 ctx->key_type = OTX2_CPT_AES_128_BIT; 430 break; 431 case 2 * AES_KEYSIZE_192: 432 ctx->key_type = OTX2_CPT_AES_192_BIT; 433 break; 434 case 2 * AES_KEYSIZE_256: 435 ctx->key_type = OTX2_CPT_AES_256_BIT; 436 break; 437 default: 438 return -EINVAL; 439 } 440 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 441} 442 443static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 444 u32 keylen, u8 cipher_type) 445{ 446 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 447 448 if (keylen != DES3_EDE_KEY_SIZE) 449 return -EINVAL; 450 451 ctx->key_len = keylen; 452 ctx->cipher_type = cipher_type; 453 ctx->enc_align_len = 8; 454 455 memcpy(ctx->enc_key, key, keylen); 456 457 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 458} 459 460static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 461 u32 keylen, u8 cipher_type) 462{ 463 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 464 465 switch (keylen) { 466 case AES_KEYSIZE_128: 467 ctx->key_type = OTX2_CPT_AES_128_BIT; 468 break; 469 case AES_KEYSIZE_192: 470 ctx->key_type = OTX2_CPT_AES_192_BIT; 471 break; 472 case AES_KEYSIZE_256: 473 ctx->key_type = OTX2_CPT_AES_256_BIT; 474 break; 475 default: 476 return -EINVAL; 477 } 478 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB) 479 ctx->enc_align_len = 16; 480 else 481 ctx->enc_align_len = 1; 482 483 ctx->key_len = keylen; 484 ctx->cipher_type = cipher_type; 485 486 memcpy(ctx->enc_key, key, keylen); 487 488 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); 489} 490 491static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, 492 const u8 *key, u32 keylen) 493{ 494 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC); 495} 496 497static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, 498 const u8 *key, u32 keylen) 499{ 500 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB); 501} 502 503static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, 504 const u8 *key, u32 keylen) 505{ 506 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC); 507} 508 509static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, 510 const u8 *key, u32 keylen) 511{ 512 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB); 513} 514 515static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx, 516 struct crypto_alg *alg) 517{ 518 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 519 ctx->fbk_cipher = 520 crypto_alloc_skcipher(alg->cra_name, 0, 521 CRYPTO_ALG_ASYNC | 522 CRYPTO_ALG_NEED_FALLBACK); 523 if (IS_ERR(ctx->fbk_cipher)) { 524 pr_err("%s() failed to allocate fallback for %s\n", 525 __func__, alg->cra_name); 526 return PTR_ERR(ctx->fbk_cipher); 527 } 528 } 529 return 0; 530} 531 532static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) 533{ 534 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); 535 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); 536 struct crypto_alg *alg = tfm->__crt_alg; 537 struct pci_dev *pdev; 538 int ret, cpu_num; 539 540 memset(ctx, 0, sizeof(*ctx)); 541 /* 542 * Additional memory for skcipher_request is 543 * allocated since the cryptd daemon uses 544 * this memory for request_ctx information 545 */ 546 crypto_skcipher_set_reqsize_dma( 547 stfm, sizeof(struct otx2_cpt_req_ctx) + 548 sizeof(struct skcipher_request)); 549 550 ret = get_se_device(&pdev, &cpu_num); 551 if (ret) 552 return ret; 553 554 ctx->pdev = pdev; 555 ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx); 556 if (ret) 557 return ret; 558 559 return cpt_skcipher_fallback_init(ctx, alg); 560} 561 562static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) 563{ 564 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 565 566 if (ctx->fbk_cipher) { 567 crypto_free_skcipher(ctx->fbk_cipher); 568 ctx->fbk_cipher = NULL; 569 } 570 cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx); 571} 572 573static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, 574 struct crypto_alg *alg) 575{ 576 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { 577 ctx->fbk_cipher = 578 crypto_alloc_aead(alg->cra_name, 0, 579 CRYPTO_ALG_ASYNC | 580 CRYPTO_ALG_NEED_FALLBACK); 581 if (IS_ERR(ctx->fbk_cipher)) { 582 pr_err("%s() failed to allocate fallback for %s\n", 583 __func__, alg->cra_name); 584 return PTR_ERR(ctx->fbk_cipher); 585 } 586 } 587 return 0; 588} 589 590static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) 591{ 592 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm); 593 struct crypto_tfm *tfm = crypto_aead_tfm(atfm); 594 struct crypto_alg *alg = tfm->__crt_alg; 595 struct pci_dev *pdev; 596 int ret, cpu_num; 597 598 ctx->cipher_type = cipher_type; 599 ctx->mac_type = mac_type; 600 601 /* 602 * When selected cipher is NULL we use HMAC opcode instead of 603 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms 604 * for calculating ipad and opad 605 */ 606 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) { 607 switch (ctx->mac_type) { 608 case OTX2_CPT_SHA1: 609 ctx->hashalg = crypto_alloc_shash("sha1", 0, 610 CRYPTO_ALG_ASYNC); 611 if (IS_ERR(ctx->hashalg)) 612 return PTR_ERR(ctx->hashalg); 613 break; 614 615 case OTX2_CPT_SHA256: 616 ctx->hashalg = crypto_alloc_shash("sha256", 0, 617 CRYPTO_ALG_ASYNC); 618 if (IS_ERR(ctx->hashalg)) 619 return PTR_ERR(ctx->hashalg); 620 break; 621 622 case OTX2_CPT_SHA384: 623 ctx->hashalg = crypto_alloc_shash("sha384", 0, 624 CRYPTO_ALG_ASYNC); 625 if (IS_ERR(ctx->hashalg)) 626 return PTR_ERR(ctx->hashalg); 627 break; 628 629 case OTX2_CPT_SHA512: 630 ctx->hashalg = crypto_alloc_shash("sha512", 0, 631 CRYPTO_ALG_ASYNC); 632 if (IS_ERR(ctx->hashalg)) 633 return PTR_ERR(ctx->hashalg); 634 break; 635 } 636 } 637 switch (ctx->cipher_type) { 638 case OTX2_CPT_AES_CBC: 639 case OTX2_CPT_AES_ECB: 640 ctx->enc_align_len = 16; 641 break; 642 case OTX2_CPT_DES3_CBC: 643 case OTX2_CPT_DES3_ECB: 644 ctx->enc_align_len = 8; 645 break; 646 case OTX2_CPT_AES_GCM: 647 case OTX2_CPT_CIPHER_NULL: 648 ctx->enc_align_len = 1; 649 break; 650 } 651 crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx)); 652 653 ret = get_se_device(&pdev, &cpu_num); 654 if (ret) 655 return ret; 656 657 ctx->pdev = pdev; 658 ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx); 659 if (ret) 660 return ret; 661 662 return cpt_aead_fallback_init(ctx, alg); 663} 664 665static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) 666{ 667 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1); 668} 669 670static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) 671{ 672 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256); 673} 674 675static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) 676{ 677 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384); 678} 679 680static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) 681{ 682 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512); 683} 684 685static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) 686{ 687 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1); 688} 689 690static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) 691{ 692 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256); 693} 694 695static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) 696{ 697 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384); 698} 699 700static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) 701{ 702 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512); 703} 704 705static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) 706{ 707 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL); 708} 709 710static void otx2_cpt_aead_exit(struct crypto_aead *tfm) 711{ 712 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 713 714 kfree(ctx->ipad); 715 kfree(ctx->opad); 716 if (ctx->hashalg) 717 crypto_free_shash(ctx->hashalg); 718 kfree(ctx->sdesc); 719 720 if (ctx->fbk_cipher) { 721 crypto_free_aead(ctx->fbk_cipher); 722 ctx->fbk_cipher = NULL; 723 } 724 cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx); 725} 726 727static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, 728 unsigned int authsize) 729{ 730 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 731 732 if (crypto_rfc4106_check_authsize(authsize)) 733 return -EINVAL; 734 735 tfm->authsize = authsize; 736 /* Set authsize for fallback case */ 737 if (ctx->fbk_cipher) 738 ctx->fbk_cipher->authsize = authsize; 739 740 return 0; 741} 742 743static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm, 744 unsigned int authsize) 745{ 746 tfm->authsize = authsize; 747 748 return 0; 749} 750 751static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm, 752 unsigned int authsize) 753{ 754 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 755 756 ctx->is_trunc_hmac = true; 757 tfm->authsize = authsize; 758 759 return 0; 760} 761 762static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) 763{ 764 struct otx2_cpt_sdesc *sdesc; 765 int size; 766 767 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); 768 sdesc = kmalloc(size, GFP_KERNEL); 769 if (!sdesc) 770 return NULL; 771 772 sdesc->shash.tfm = alg; 773 774 return sdesc; 775} 776 777static inline void swap_data32(void *buf, u32 len) 778{ 779 cpu_to_be32_array(buf, buf, len / 4); 780} 781 782static inline void swap_data64(void *buf, u32 len) 783{ 784 u64 *src = buf; 785 int i = 0; 786 787 for (i = 0 ; i < len / 8; i++, src++) 788 cpu_to_be64s(src); 789} 790 791static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad) 792{ 793 struct sha512_state *sha512; 794 struct sha256_state *sha256; 795 struct sha1_state *sha1; 796 797 switch (mac_type) { 798 case OTX2_CPT_SHA1: 799 sha1 = (struct sha1_state *) in_pad; 800 swap_data32(sha1->state, SHA1_DIGEST_SIZE); 801 memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE); 802 break; 803 804 case OTX2_CPT_SHA256: 805 sha256 = (struct sha256_state *) in_pad; 806 swap_data32(sha256->state, SHA256_DIGEST_SIZE); 807 memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE); 808 break; 809 810 case OTX2_CPT_SHA384: 811 case OTX2_CPT_SHA512: 812 sha512 = (struct sha512_state *) in_pad; 813 swap_data64(sha512->state, SHA512_DIGEST_SIZE); 814 memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE); 815 break; 816 817 default: 818 return -EINVAL; 819 } 820 821 return 0; 822} 823 824static int aead_hmac_init(struct crypto_aead *cipher) 825{ 826 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 827 int state_size = crypto_shash_statesize(ctx->hashalg); 828 int ds = crypto_shash_digestsize(ctx->hashalg); 829 int bs = crypto_shash_blocksize(ctx->hashalg); 830 int authkeylen = ctx->auth_key_len; 831 u8 *ipad = NULL, *opad = NULL; 832 int ret = 0, icount = 0; 833 834 ctx->sdesc = alloc_sdesc(ctx->hashalg); 835 if (!ctx->sdesc) 836 return -ENOMEM; 837 838 ctx->ipad = kzalloc(bs, GFP_KERNEL); 839 if (!ctx->ipad) { 840 ret = -ENOMEM; 841 goto calc_fail; 842 } 843 844 ctx->opad = kzalloc(bs, GFP_KERNEL); 845 if (!ctx->opad) { 846 ret = -ENOMEM; 847 goto calc_fail; 848 } 849 850 ipad = kzalloc(state_size, GFP_KERNEL); 851 if (!ipad) { 852 ret = -ENOMEM; 853 goto calc_fail; 854 } 855 856 opad = kzalloc(state_size, GFP_KERNEL); 857 if (!opad) { 858 ret = -ENOMEM; 859 goto calc_fail; 860 } 861 862 if (authkeylen > bs) { 863 ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key, 864 authkeylen, ipad); 865 if (ret) 866 goto calc_fail; 867 868 authkeylen = ds; 869 } else { 870 memcpy(ipad, ctx->key, authkeylen); 871 } 872 873 memset(ipad + authkeylen, 0, bs - authkeylen); 874 memcpy(opad, ipad, bs); 875 876 for (icount = 0; icount < bs; icount++) { 877 ipad[icount] ^= 0x36; 878 opad[icount] ^= 0x5c; 879 } 880 881 /* 882 * Partial Hash calculated from the software 883 * algorithm is retrieved for IPAD & OPAD 884 */ 885 886 /* IPAD Calculation */ 887 crypto_shash_init(&ctx->sdesc->shash); 888 crypto_shash_update(&ctx->sdesc->shash, ipad, bs); 889 crypto_shash_export(&ctx->sdesc->shash, ipad); 890 ret = copy_pad(ctx->mac_type, ctx->ipad, ipad); 891 if (ret) 892 goto calc_fail; 893 894 /* OPAD Calculation */ 895 crypto_shash_init(&ctx->sdesc->shash); 896 crypto_shash_update(&ctx->sdesc->shash, opad, bs); 897 crypto_shash_export(&ctx->sdesc->shash, opad); 898 ret = copy_pad(ctx->mac_type, ctx->opad, opad); 899 if (ret) 900 goto calc_fail; 901 902 kfree(ipad); 903 kfree(opad); 904 905 return 0; 906 907calc_fail: 908 kfree(ctx->ipad); 909 ctx->ipad = NULL; 910 kfree(ctx->opad); 911 ctx->opad = NULL; 912 kfree(ipad); 913 kfree(opad); 914 kfree(ctx->sdesc); 915 ctx->sdesc = NULL; 916 917 return ret; 918} 919 920static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, 921 const unsigned char *key, 922 unsigned int keylen) 923{ 924 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 925 struct crypto_authenc_key_param *param; 926 int enckeylen = 0, authkeylen = 0; 927 struct rtattr *rta = (void *)key; 928 929 if (!RTA_OK(rta, keylen)) 930 return -EINVAL; 931 932 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 933 return -EINVAL; 934 935 if (RTA_PAYLOAD(rta) < sizeof(*param)) 936 return -EINVAL; 937 938 param = RTA_DATA(rta); 939 enckeylen = be32_to_cpu(param->enckeylen); 940 key += RTA_ALIGN(rta->rta_len); 941 keylen -= RTA_ALIGN(rta->rta_len); 942 if (keylen < enckeylen) 943 return -EINVAL; 944 945 if (keylen > OTX2_CPT_MAX_KEY_SIZE) 946 return -EINVAL; 947 948 authkeylen = keylen - enckeylen; 949 memcpy(ctx->key, key, keylen); 950 951 switch (enckeylen) { 952 case AES_KEYSIZE_128: 953 ctx->key_type = OTX2_CPT_AES_128_BIT; 954 break; 955 case AES_KEYSIZE_192: 956 ctx->key_type = OTX2_CPT_AES_192_BIT; 957 break; 958 case AES_KEYSIZE_256: 959 ctx->key_type = OTX2_CPT_AES_256_BIT; 960 break; 961 default: 962 /* Invalid key length */ 963 return -EINVAL; 964 } 965 966 ctx->enc_key_len = enckeylen; 967 ctx->auth_key_len = authkeylen; 968 969 return aead_hmac_init(cipher); 970} 971 972static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, 973 const unsigned char *key, 974 unsigned int keylen) 975{ 976 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 977 struct crypto_authenc_key_param *param; 978 struct rtattr *rta = (void *)key; 979 int enckeylen = 0; 980 981 if (!RTA_OK(rta, keylen)) 982 return -EINVAL; 983 984 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 985 return -EINVAL; 986 987 if (RTA_PAYLOAD(rta) < sizeof(*param)) 988 return -EINVAL; 989 990 param = RTA_DATA(rta); 991 enckeylen = be32_to_cpu(param->enckeylen); 992 key += RTA_ALIGN(rta->rta_len); 993 keylen -= RTA_ALIGN(rta->rta_len); 994 if (enckeylen != 0) 995 return -EINVAL; 996 997 if (keylen > OTX2_CPT_MAX_KEY_SIZE) 998 return -EINVAL; 999 1000 memcpy(ctx->key, key, keylen); 1001 ctx->enc_key_len = enckeylen; 1002 ctx->auth_key_len = keylen; 1003 1004 return 0; 1005} 1006 1007static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, 1008 const unsigned char *key, 1009 unsigned int keylen) 1010{ 1011 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 1012 1013 /* 1014 * For aes gcm we expect to get encryption key (16, 24, 32 bytes) 1015 * and salt (4 bytes) 1016 */ 1017 switch (keylen) { 1018 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: 1019 ctx->key_type = OTX2_CPT_AES_128_BIT; 1020 ctx->enc_key_len = AES_KEYSIZE_128; 1021 break; 1022 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: 1023 ctx->key_type = OTX2_CPT_AES_192_BIT; 1024 ctx->enc_key_len = AES_KEYSIZE_192; 1025 break; 1026 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: 1027 ctx->key_type = OTX2_CPT_AES_256_BIT; 1028 ctx->enc_key_len = AES_KEYSIZE_256; 1029 break; 1030 default: 1031 /* Invalid key and salt length */ 1032 return -EINVAL; 1033 } 1034 1035 /* Store encryption key and salt */ 1036 memcpy(ctx->key, key, keylen); 1037 1038 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen); 1039} 1040 1041static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc, 1042 u32 *argcnt) 1043{ 1044 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1045 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1046 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 1047 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1048 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; 1049 int mac_len = crypto_aead_authsize(tfm); 1050 int ds; 1051 1052 rctx->ctrl_word.e.enc_data_offset = req->assoclen; 1053 1054 switch (ctx->cipher_type) { 1055 case OTX2_CPT_AES_CBC: 1056 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8)) 1057 return -EINVAL; 1058 1059 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; 1060 /* Copy encryption key to context */ 1061 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, 1062 ctx->enc_key_len); 1063 /* Copy IV to context */ 1064 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); 1065 1066 ds = crypto_shash_digestsize(ctx->hashalg); 1067 if (ctx->mac_type == OTX2_CPT_SHA384) 1068 ds = SHA512_DIGEST_SIZE; 1069 if (ctx->ipad) 1070 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); 1071 if (ctx->opad) 1072 memcpy(fctx->hmac.e.opad, ctx->opad, ds); 1073 break; 1074 1075 case OTX2_CPT_AES_GCM: 1076 if (crypto_ipsec_check_assoclen(req->assoclen)) 1077 return -EINVAL; 1078 1079 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR; 1080 /* Copy encryption key to context */ 1081 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); 1082 /* Copy salt to context */ 1083 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, 1084 AES_GCM_SALT_SIZE); 1085 1086 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; 1087 break; 1088 1089 default: 1090 /* Unknown cipher type */ 1091 return -EINVAL; 1092 } 1093 cpu_to_be64s(&rctx->ctrl_word.flags); 1094 1095 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 1096 req_info->ctrl.s.se_req = 1; 1097 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | 1098 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 1099 if (enc) { 1100 req_info->req.opcode.s.minor = 2; 1101 req_info->req.param1 = req->cryptlen; 1102 req_info->req.param2 = req->cryptlen + req->assoclen; 1103 } else { 1104 req_info->req.opcode.s.minor = 3; 1105 req_info->req.param1 = req->cryptlen - mac_len; 1106 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; 1107 } 1108 1109 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 1110 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 1111 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; 1112 fctx->enc.enc_ctrl.e.mac_len = mac_len; 1113 cpu_to_be64s(&fctx->enc.enc_ctrl.u); 1114 1115 /* 1116 * Storing Packet Data Information in offset 1117 * Control Word First 8 bytes 1118 */ 1119 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 1120 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 1121 req_info->req.dlen += CONTROL_WORD_LEN; 1122 ++(*argcnt); 1123 1124 req_info->in[*argcnt].vptr = (u8 *)fctx; 1125 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); 1126 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); 1127 ++(*argcnt); 1128 1129 return 0; 1130} 1131 1132static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, 1133 u32 enc) 1134{ 1135 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1136 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1137 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 1138 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1139 1140 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; 1141 req_info->ctrl.s.se_req = 1; 1142 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC | 1143 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); 1144 req_info->is_trunc_hmac = ctx->is_trunc_hmac; 1145 1146 req_info->req.opcode.s.minor = 0; 1147 req_info->req.param1 = ctx->auth_key_len; 1148 req_info->req.param2 = ctx->mac_type << 8; 1149 1150 /* Add authentication key */ 1151 req_info->in[*argcnt].vptr = ctx->key; 1152 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); 1153 req_info->req.dlen += round_up(ctx->auth_key_len, 8); 1154 ++(*argcnt); 1155} 1156 1157static inline int create_aead_input_list(struct aead_request *req, u32 enc) 1158{ 1159 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1160 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1161 u32 inputlen = req->cryptlen + req->assoclen; 1162 u32 status, argcnt = 0; 1163 1164 status = create_aead_ctx_hdr(req, enc, &argcnt); 1165 if (status) 1166 return status; 1167 update_input_data(req_info, req->src, inputlen, &argcnt); 1168 req_info->in_cnt = argcnt; 1169 1170 return 0; 1171} 1172 1173static inline void create_aead_output_list(struct aead_request *req, u32 enc, 1174 u32 mac_len) 1175{ 1176 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1177 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1178 u32 argcnt = 0, outputlen = 0; 1179 1180 if (enc) 1181 outputlen = req->cryptlen + req->assoclen + mac_len; 1182 else 1183 outputlen = req->cryptlen + req->assoclen - mac_len; 1184 1185 update_output_data(req_info, req->dst, 0, outputlen, &argcnt); 1186 req_info->out_cnt = argcnt; 1187} 1188 1189static inline void create_aead_null_input_list(struct aead_request *req, 1190 u32 enc, u32 mac_len) 1191{ 1192 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1193 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1194 u32 inputlen, argcnt = 0; 1195 1196 if (enc) 1197 inputlen = req->cryptlen + req->assoclen; 1198 else 1199 inputlen = req->cryptlen + req->assoclen - mac_len; 1200 1201 create_hmac_ctx_hdr(req, &argcnt, enc); 1202 update_input_data(req_info, req->src, inputlen, &argcnt); 1203 req_info->in_cnt = argcnt; 1204} 1205 1206static inline int create_aead_null_output_list(struct aead_request *req, 1207 u32 enc, u32 mac_len) 1208{ 1209 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1210 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1211 struct scatterlist *dst; 1212 u8 *ptr = NULL; 1213 int argcnt = 0, status, offset; 1214 u32 inputlen; 1215 1216 if (enc) 1217 inputlen = req->cryptlen + req->assoclen; 1218 else 1219 inputlen = req->cryptlen + req->assoclen - mac_len; 1220 1221 /* 1222 * If source and destination are different 1223 * then copy payload to destination 1224 */ 1225 if (req->src != req->dst) { 1226 1227 ptr = kmalloc(inputlen, (req_info->areq->flags & 1228 CRYPTO_TFM_REQ_MAY_SLEEP) ? 1229 GFP_KERNEL : GFP_ATOMIC); 1230 if (!ptr) 1231 return -ENOMEM; 1232 1233 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, 1234 inputlen); 1235 if (status != inputlen) { 1236 status = -EINVAL; 1237 goto error_free; 1238 } 1239 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, 1240 inputlen); 1241 if (status != inputlen) { 1242 status = -EINVAL; 1243 goto error_free; 1244 } 1245 kfree(ptr); 1246 } 1247 1248 if (enc) { 1249 /* 1250 * In an encryption scenario hmac needs 1251 * to be appended after payload 1252 */ 1253 dst = req->dst; 1254 offset = inputlen; 1255 while (offset >= dst->length) { 1256 offset -= dst->length; 1257 dst = sg_next(dst); 1258 if (!dst) 1259 return -ENOENT; 1260 } 1261 1262 update_output_data(req_info, dst, offset, mac_len, &argcnt); 1263 } else { 1264 /* 1265 * In a decryption scenario calculated hmac for received 1266 * payload needs to be compare with hmac received 1267 */ 1268 status = sg_copy_buffer(req->src, sg_nents(req->src), 1269 rctx->fctx.hmac.s.hmac_recv, mac_len, 1270 inputlen, true); 1271 if (status != mac_len) 1272 return -EINVAL; 1273 1274 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; 1275 req_info->out[argcnt].size = mac_len; 1276 argcnt++; 1277 } 1278 1279 req_info->out_cnt = argcnt; 1280 return 0; 1281 1282error_free: 1283 kfree(ptr); 1284 return status; 1285} 1286 1287static int aead_do_fallback(struct aead_request *req, bool is_enc) 1288{ 1289 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1290 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1291 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead); 1292 int ret; 1293 1294 if (ctx->fbk_cipher) { 1295 /* Store the cipher tfm and then use the fallback tfm */ 1296 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher); 1297 aead_request_set_callback(&rctx->fbk_req, req->base.flags, 1298 req->base.complete, req->base.data); 1299 aead_request_set_crypt(&rctx->fbk_req, req->src, 1300 req->dst, req->cryptlen, req->iv); 1301 aead_request_set_ad(&rctx->fbk_req, req->assoclen); 1302 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : 1303 crypto_aead_decrypt(&rctx->fbk_req); 1304 } else { 1305 ret = -EINVAL; 1306 } 1307 1308 return ret; 1309} 1310 1311static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) 1312{ 1313 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1314 struct otx2_cpt_req_info *req_info = &rctx->cpt_req; 1315 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1316 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 1317 struct pci_dev *pdev; 1318 int status, cpu_num; 1319 1320 /* Clear control words */ 1321 rctx->ctrl_word.flags = 0; 1322 rctx->fctx.enc.enc_ctrl.u = 0; 1323 1324 req_info->callback = otx2_cpt_aead_callback; 1325 req_info->areq = &req->base; 1326 req_info->req_type = reg_type; 1327 req_info->is_enc = enc; 1328 req_info->is_trunc_hmac = false; 1329 1330 req_info->req.cptr = ctx->er_ctx.hw_ctx; 1331 req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; 1332 1333 switch (reg_type) { 1334 case OTX2_CPT_AEAD_ENC_DEC_REQ: 1335 status = create_aead_input_list(req, enc); 1336 if (status) 1337 return status; 1338 create_aead_output_list(req, enc, crypto_aead_authsize(tfm)); 1339 break; 1340 1341 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ: 1342 create_aead_null_input_list(req, enc, 1343 crypto_aead_authsize(tfm)); 1344 status = create_aead_null_output_list(req, enc, 1345 crypto_aead_authsize(tfm)); 1346 if (status) 1347 return status; 1348 break; 1349 1350 default: 1351 return -EINVAL; 1352 } 1353 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len)) 1354 return -EINVAL; 1355 1356 if (!req_info->req.param2 || 1357 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) || 1358 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE)) 1359 return aead_do_fallback(req, enc); 1360 1361 status = get_se_device(&pdev, &cpu_num); 1362 if (status) 1363 return status; 1364 1365 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); 1366 1367 /* 1368 * We perform an asynchronous send and once 1369 * the request is completed the driver would 1370 * intimate through registered call back functions 1371 */ 1372 return otx2_cpt_do_request(pdev, req_info, cpu_num); 1373} 1374 1375static int otx2_cpt_aead_encrypt(struct aead_request *req) 1376{ 1377 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true); 1378} 1379 1380static int otx2_cpt_aead_decrypt(struct aead_request *req) 1381{ 1382 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false); 1383} 1384 1385static int otx2_cpt_aead_null_encrypt(struct aead_request *req) 1386{ 1387 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true); 1388} 1389 1390static int otx2_cpt_aead_null_decrypt(struct aead_request *req) 1391{ 1392 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false); 1393} 1394 1395static struct skcipher_alg otx2_cpt_skciphers[] = { { 1396 .base.cra_name = "xts(aes)", 1397 .base.cra_driver_name = "cpt_xts_aes", 1398 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1399 .base.cra_blocksize = AES_BLOCK_SIZE, 1400 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1401 .base.cra_alignmask = 7, 1402 .base.cra_priority = 4001, 1403 .base.cra_module = THIS_MODULE, 1404 1405 .init = otx2_cpt_enc_dec_init, 1406 .exit = otx2_cpt_skcipher_exit, 1407 .ivsize = AES_BLOCK_SIZE, 1408 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1409 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1410 .setkey = otx2_cpt_skcipher_xts_setkey, 1411 .encrypt = otx2_cpt_skcipher_encrypt, 1412 .decrypt = otx2_cpt_skcipher_decrypt, 1413}, { 1414 .base.cra_name = "cbc(aes)", 1415 .base.cra_driver_name = "cpt_cbc_aes", 1416 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1417 .base.cra_blocksize = AES_BLOCK_SIZE, 1418 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1419 .base.cra_alignmask = 7, 1420 .base.cra_priority = 4001, 1421 .base.cra_module = THIS_MODULE, 1422 1423 .init = otx2_cpt_enc_dec_init, 1424 .exit = otx2_cpt_skcipher_exit, 1425 .ivsize = AES_BLOCK_SIZE, 1426 .min_keysize = AES_MIN_KEY_SIZE, 1427 .max_keysize = AES_MAX_KEY_SIZE, 1428 .setkey = otx2_cpt_skcipher_cbc_aes_setkey, 1429 .encrypt = otx2_cpt_skcipher_encrypt, 1430 .decrypt = otx2_cpt_skcipher_decrypt, 1431}, { 1432 .base.cra_name = "ecb(aes)", 1433 .base.cra_driver_name = "cpt_ecb_aes", 1434 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1435 .base.cra_blocksize = AES_BLOCK_SIZE, 1436 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1437 .base.cra_alignmask = 7, 1438 .base.cra_priority = 4001, 1439 .base.cra_module = THIS_MODULE, 1440 1441 .init = otx2_cpt_enc_dec_init, 1442 .exit = otx2_cpt_skcipher_exit, 1443 .ivsize = 0, 1444 .min_keysize = AES_MIN_KEY_SIZE, 1445 .max_keysize = AES_MAX_KEY_SIZE, 1446 .setkey = otx2_cpt_skcipher_ecb_aes_setkey, 1447 .encrypt = otx2_cpt_skcipher_encrypt, 1448 .decrypt = otx2_cpt_skcipher_decrypt, 1449}, { 1450 .base.cra_name = "cbc(des3_ede)", 1451 .base.cra_driver_name = "cpt_cbc_des3_ede", 1452 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1453 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1454 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1455 .base.cra_alignmask = 7, 1456 .base.cra_priority = 4001, 1457 .base.cra_module = THIS_MODULE, 1458 1459 .init = otx2_cpt_enc_dec_init, 1460 .exit = otx2_cpt_skcipher_exit, 1461 .min_keysize = DES3_EDE_KEY_SIZE, 1462 .max_keysize = DES3_EDE_KEY_SIZE, 1463 .ivsize = DES_BLOCK_SIZE, 1464 .setkey = otx2_cpt_skcipher_cbc_des3_setkey, 1465 .encrypt = otx2_cpt_skcipher_encrypt, 1466 .decrypt = otx2_cpt_skcipher_decrypt, 1467}, { 1468 .base.cra_name = "ecb(des3_ede)", 1469 .base.cra_driver_name = "cpt_ecb_des3_ede", 1470 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1471 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1472 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), 1473 .base.cra_alignmask = 7, 1474 .base.cra_priority = 4001, 1475 .base.cra_module = THIS_MODULE, 1476 1477 .init = otx2_cpt_enc_dec_init, 1478 .exit = otx2_cpt_skcipher_exit, 1479 .min_keysize = DES3_EDE_KEY_SIZE, 1480 .max_keysize = DES3_EDE_KEY_SIZE, 1481 .ivsize = 0, 1482 .setkey = otx2_cpt_skcipher_ecb_des3_setkey, 1483 .encrypt = otx2_cpt_skcipher_encrypt, 1484 .decrypt = otx2_cpt_skcipher_decrypt, 1485} }; 1486 1487static struct aead_alg otx2_cpt_aeads[] = { { 1488 .base = { 1489 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1490 .cra_driver_name = "cpt_hmac_sha1_cbc_aes", 1491 .cra_blocksize = AES_BLOCK_SIZE, 1492 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1493 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1494 .cra_priority = 4001, 1495 .cra_alignmask = 0, 1496 .cra_module = THIS_MODULE, 1497 }, 1498 .init = otx2_cpt_aead_cbc_aes_sha1_init, 1499 .exit = otx2_cpt_aead_exit, 1500 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1501 .setauthsize = otx2_cpt_aead_set_authsize, 1502 .encrypt = otx2_cpt_aead_encrypt, 1503 .decrypt = otx2_cpt_aead_decrypt, 1504 .ivsize = AES_BLOCK_SIZE, 1505 .maxauthsize = SHA1_DIGEST_SIZE, 1506}, { 1507 .base = { 1508 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1509 .cra_driver_name = "cpt_hmac_sha256_cbc_aes", 1510 .cra_blocksize = AES_BLOCK_SIZE, 1511 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1512 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1513 .cra_priority = 4001, 1514 .cra_alignmask = 0, 1515 .cra_module = THIS_MODULE, 1516 }, 1517 .init = otx2_cpt_aead_cbc_aes_sha256_init, 1518 .exit = otx2_cpt_aead_exit, 1519 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1520 .setauthsize = otx2_cpt_aead_set_authsize, 1521 .encrypt = otx2_cpt_aead_encrypt, 1522 .decrypt = otx2_cpt_aead_decrypt, 1523 .ivsize = AES_BLOCK_SIZE, 1524 .maxauthsize = SHA256_DIGEST_SIZE, 1525}, { 1526 .base = { 1527 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1528 .cra_driver_name = "cpt_hmac_sha384_cbc_aes", 1529 .cra_blocksize = AES_BLOCK_SIZE, 1530 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1531 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1532 .cra_priority = 4001, 1533 .cra_alignmask = 0, 1534 .cra_module = THIS_MODULE, 1535 }, 1536 .init = otx2_cpt_aead_cbc_aes_sha384_init, 1537 .exit = otx2_cpt_aead_exit, 1538 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1539 .setauthsize = otx2_cpt_aead_set_authsize, 1540 .encrypt = otx2_cpt_aead_encrypt, 1541 .decrypt = otx2_cpt_aead_decrypt, 1542 .ivsize = AES_BLOCK_SIZE, 1543 .maxauthsize = SHA384_DIGEST_SIZE, 1544}, { 1545 .base = { 1546 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1547 .cra_driver_name = "cpt_hmac_sha512_cbc_aes", 1548 .cra_blocksize = AES_BLOCK_SIZE, 1549 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1550 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1551 .cra_priority = 4001, 1552 .cra_alignmask = 0, 1553 .cra_module = THIS_MODULE, 1554 }, 1555 .init = otx2_cpt_aead_cbc_aes_sha512_init, 1556 .exit = otx2_cpt_aead_exit, 1557 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, 1558 .setauthsize = otx2_cpt_aead_set_authsize, 1559 .encrypt = otx2_cpt_aead_encrypt, 1560 .decrypt = otx2_cpt_aead_decrypt, 1561 .ivsize = AES_BLOCK_SIZE, 1562 .maxauthsize = SHA512_DIGEST_SIZE, 1563}, { 1564 .base = { 1565 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", 1566 .cra_driver_name = "cpt_hmac_sha1_ecb_null", 1567 .cra_blocksize = 1, 1568 .cra_flags = CRYPTO_ALG_ASYNC, 1569 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1570 .cra_priority = 4001, 1571 .cra_alignmask = 0, 1572 .cra_module = THIS_MODULE, 1573 }, 1574 .init = otx2_cpt_aead_ecb_null_sha1_init, 1575 .exit = otx2_cpt_aead_exit, 1576 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1577 .setauthsize = otx2_cpt_aead_null_set_authsize, 1578 .encrypt = otx2_cpt_aead_null_encrypt, 1579 .decrypt = otx2_cpt_aead_null_decrypt, 1580 .ivsize = 0, 1581 .maxauthsize = SHA1_DIGEST_SIZE, 1582}, { 1583 .base = { 1584 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", 1585 .cra_driver_name = "cpt_hmac_sha256_ecb_null", 1586 .cra_blocksize = 1, 1587 .cra_flags = CRYPTO_ALG_ASYNC, 1588 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1589 .cra_priority = 4001, 1590 .cra_alignmask = 0, 1591 .cra_module = THIS_MODULE, 1592 }, 1593 .init = otx2_cpt_aead_ecb_null_sha256_init, 1594 .exit = otx2_cpt_aead_exit, 1595 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1596 .setauthsize = otx2_cpt_aead_null_set_authsize, 1597 .encrypt = otx2_cpt_aead_null_encrypt, 1598 .decrypt = otx2_cpt_aead_null_decrypt, 1599 .ivsize = 0, 1600 .maxauthsize = SHA256_DIGEST_SIZE, 1601}, { 1602 .base = { 1603 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", 1604 .cra_driver_name = "cpt_hmac_sha384_ecb_null", 1605 .cra_blocksize = 1, 1606 .cra_flags = CRYPTO_ALG_ASYNC, 1607 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1608 .cra_priority = 4001, 1609 .cra_alignmask = 0, 1610 .cra_module = THIS_MODULE, 1611 }, 1612 .init = otx2_cpt_aead_ecb_null_sha384_init, 1613 .exit = otx2_cpt_aead_exit, 1614 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1615 .setauthsize = otx2_cpt_aead_null_set_authsize, 1616 .encrypt = otx2_cpt_aead_null_encrypt, 1617 .decrypt = otx2_cpt_aead_null_decrypt, 1618 .ivsize = 0, 1619 .maxauthsize = SHA384_DIGEST_SIZE, 1620}, { 1621 .base = { 1622 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", 1623 .cra_driver_name = "cpt_hmac_sha512_ecb_null", 1624 .cra_blocksize = 1, 1625 .cra_flags = CRYPTO_ALG_ASYNC, 1626 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1627 .cra_priority = 4001, 1628 .cra_alignmask = 0, 1629 .cra_module = THIS_MODULE, 1630 }, 1631 .init = otx2_cpt_aead_ecb_null_sha512_init, 1632 .exit = otx2_cpt_aead_exit, 1633 .setkey = otx2_cpt_aead_ecb_null_sha_setkey, 1634 .setauthsize = otx2_cpt_aead_null_set_authsize, 1635 .encrypt = otx2_cpt_aead_null_encrypt, 1636 .decrypt = otx2_cpt_aead_null_decrypt, 1637 .ivsize = 0, 1638 .maxauthsize = SHA512_DIGEST_SIZE, 1639}, { 1640 .base = { 1641 .cra_name = "rfc4106(gcm(aes))", 1642 .cra_driver_name = "cpt_rfc4106_gcm_aes", 1643 .cra_blocksize = 1, 1644 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 1645 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1646 .cra_priority = 4001, 1647 .cra_alignmask = 0, 1648 .cra_module = THIS_MODULE, 1649 }, 1650 .init = otx2_cpt_aead_gcm_aes_init, 1651 .exit = otx2_cpt_aead_exit, 1652 .setkey = otx2_cpt_aead_gcm_aes_setkey, 1653 .setauthsize = otx2_cpt_aead_gcm_set_authsize, 1654 .encrypt = otx2_cpt_aead_encrypt, 1655 .decrypt = otx2_cpt_aead_decrypt, 1656 .ivsize = AES_GCM_IV_SIZE, 1657 .maxauthsize = AES_GCM_ICV_SIZE, 1658} }; 1659 1660static inline int cpt_register_algs(void) 1661{ 1662 int i, err = 0; 1663 1664 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++) 1665 otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1666 1667 err = crypto_register_skciphers(otx2_cpt_skciphers, 1668 ARRAY_SIZE(otx2_cpt_skciphers)); 1669 if (err) 1670 return err; 1671 1672 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++) 1673 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1674 1675 err = crypto_register_aeads(otx2_cpt_aeads, 1676 ARRAY_SIZE(otx2_cpt_aeads)); 1677 if (err) { 1678 crypto_unregister_skciphers(otx2_cpt_skciphers, 1679 ARRAY_SIZE(otx2_cpt_skciphers)); 1680 return err; 1681 } 1682 1683 return 0; 1684} 1685 1686static inline void cpt_unregister_algs(void) 1687{ 1688 crypto_unregister_skciphers(otx2_cpt_skciphers, 1689 ARRAY_SIZE(otx2_cpt_skciphers)); 1690 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads)); 1691} 1692 1693static int compare_func(const void *lptr, const void *rptr) 1694{ 1695 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; 1696 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; 1697 1698 if (ldesc->dev->devfn < rdesc->dev->devfn) 1699 return -1; 1700 if (ldesc->dev->devfn > rdesc->dev->devfn) 1701 return 1; 1702 return 0; 1703} 1704 1705static void swap_func(void *lptr, void *rptr, int size) 1706{ 1707 struct cpt_device_desc *ldesc = lptr; 1708 struct cpt_device_desc *rdesc = rptr; 1709 1710 swap(*ldesc, *rdesc); 1711} 1712 1713int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, 1714 int num_queues, int num_devices) 1715{ 1716 int ret = 0; 1717 int count; 1718 1719 mutex_lock(&mutex); 1720 count = atomic_read(&se_devices.count); 1721 if (count >= OTX2_CPT_MAX_LFS_NUM) { 1722 dev_err(&pdev->dev, "No space to add a new device\n"); 1723 ret = -ENOSPC; 1724 goto unlock; 1725 } 1726 se_devices.desc[count].num_queues = num_queues; 1727 se_devices.desc[count++].dev = pdev; 1728 atomic_inc(&se_devices.count); 1729 1730 if (atomic_read(&se_devices.count) == num_devices && 1731 is_crypto_registered == false) { 1732 if (cpt_register_algs()) { 1733 dev_err(&pdev->dev, 1734 "Error in registering crypto algorithms\n"); 1735 ret = -EINVAL; 1736 goto unlock; 1737 } 1738 try_module_get(mod); 1739 is_crypto_registered = true; 1740 } 1741 sort(se_devices.desc, count, sizeof(struct cpt_device_desc), 1742 compare_func, swap_func); 1743 1744unlock: 1745 mutex_unlock(&mutex); 1746 return ret; 1747} 1748 1749void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod) 1750{ 1751 struct cpt_device_table *dev_tbl; 1752 bool dev_found = false; 1753 int i, j, count; 1754 1755 mutex_lock(&mutex); 1756 1757 dev_tbl = &se_devices; 1758 count = atomic_read(&dev_tbl->count); 1759 for (i = 0; i < count; i++) { 1760 if (pdev == dev_tbl->desc[i].dev) { 1761 for (j = i; j < count-1; j++) 1762 dev_tbl->desc[j] = dev_tbl->desc[j+1]; 1763 dev_found = true; 1764 break; 1765 } 1766 } 1767 1768 if (!dev_found) { 1769 dev_err(&pdev->dev, "%s device not found\n", __func__); 1770 goto unlock; 1771 } 1772 if (atomic_dec_and_test(&se_devices.count)) { 1773 cpt_unregister_algs(); 1774 module_put(mod); 1775 is_crypto_registered = false; 1776 } 1777 1778unlock: 1779 mutex_unlock(&mutex); 1780} 1781