1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4#include <linux/kernel.h> 5#include <linux/module.h> 6#include <crypto/algapi.h> 7#include <crypto/internal/skcipher.h> 8#include <crypto/internal/des.h> 9#include <crypto/xts.h> 10#include <crypto/sm4.h> 11#include <crypto/scatterwalk.h> 12 13#include "cc_driver.h" 14#include "cc_lli_defs.h" 15#include "cc_buffer_mgr.h" 16#include "cc_cipher.h" 17#include "cc_request_mgr.h" 18 19#define MAX_SKCIPHER_SEQ_LEN 6 20 21#define template_skcipher template_u.skcipher 22 23struct cc_user_key_info { 24 u8 *key; 25 dma_addr_t key_dma_addr; 26}; 27 28struct cc_hw_key_info { 29 enum cc_hw_crypto_key key1_slot; 30 enum cc_hw_crypto_key key2_slot; 31}; 32 33struct cc_cpp_key_info { 34 u8 slot; 35 enum cc_cpp_alg alg; 36}; 37 38enum cc_key_type { 39 CC_UNPROTECTED_KEY, /* User key */ 40 CC_HW_PROTECTED_KEY, /* HW (FDE) key */ 41 CC_POLICY_PROTECTED_KEY, /* CPP key */ 42 CC_INVALID_PROTECTED_KEY /* Invalid key */ 43}; 44 45struct cc_cipher_ctx { 46 struct cc_drvdata *drvdata; 47 int keylen; 48 int cipher_mode; 49 int flow_mode; 50 unsigned int flags; 51 enum cc_key_type key_type; 52 struct cc_user_key_info user; 53 union { 54 struct cc_hw_key_info hw; 55 struct cc_cpp_key_info cpp; 56 }; 57 struct crypto_shash *shash_tfm; 58 struct crypto_skcipher *fallback_tfm; 59 bool fallback_on; 60}; 61 62static void cc_cipher_complete(struct device *dev, void *cc_req, int err); 63 64static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm) 65{ 66 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 67 68 return ctx_p->key_type; 69} 70 71static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) 72{ 73 switch (ctx_p->flow_mode) { 74 case S_DIN_to_AES: 75 switch (size) { 76 case CC_AES_128_BIT_KEY_SIZE: 77 case CC_AES_192_BIT_KEY_SIZE: 78 if (ctx_p->cipher_mode != DRV_CIPHER_XTS) 79 return 0; 80 break; 81 case CC_AES_256_BIT_KEY_SIZE: 82 return 0; 83 case (CC_AES_192_BIT_KEY_SIZE * 2): 84 case (CC_AES_256_BIT_KEY_SIZE * 2): 85 if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 86 ctx_p->cipher_mode == DRV_CIPHER_ESSIV) 87 return 0; 88 break; 89 default: 90 break; 91 } 92 break; 93 case S_DIN_to_DES: 94 if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE) 95 return 0; 96 break; 97 case S_DIN_to_SM4: 98 if (size == SM4_KEY_SIZE) 99 return 0; 100 break; 101 default: 102 break; 103 } 104 return -EINVAL; 105} 106 107static int validate_data_size(struct cc_cipher_ctx *ctx_p, 108 unsigned int size) 109{ 110 switch (ctx_p->flow_mode) { 111 case S_DIN_to_AES: 112 switch (ctx_p->cipher_mode) { 113 case DRV_CIPHER_XTS: 114 case DRV_CIPHER_CBC_CTS: 115 if (size >= AES_BLOCK_SIZE) 116 return 0; 117 break; 118 case DRV_CIPHER_OFB: 119 case DRV_CIPHER_CTR: 120 return 0; 121 case DRV_CIPHER_ECB: 122 case DRV_CIPHER_CBC: 123 case DRV_CIPHER_ESSIV: 124 if (IS_ALIGNED(size, AES_BLOCK_SIZE)) 125 return 0; 126 break; 127 default: 128 break; 129 } 130 break; 131 case S_DIN_to_DES: 132 if (IS_ALIGNED(size, DES_BLOCK_SIZE)) 133 return 0; 134 break; 135 case S_DIN_to_SM4: 136 switch (ctx_p->cipher_mode) { 137 case DRV_CIPHER_CTR: 138 return 0; 139 case DRV_CIPHER_ECB: 140 case DRV_CIPHER_CBC: 141 if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) 142 return 0; 143 break; 144 default: 145 break; 146 } 147 break; 148 default: 149 break; 150 } 151 return -EINVAL; 152} 153 154static int cc_cipher_init(struct crypto_tfm *tfm) 155{ 156 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 157 struct cc_crypto_alg *cc_alg = 158 container_of(tfm->__crt_alg, struct cc_crypto_alg, 159 skcipher_alg.base); 160 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 161 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 162 unsigned int fallback_req_size = 0; 163 164 dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p, 165 crypto_tfm_alg_name(tfm)); 166 167 ctx_p->cipher_mode = cc_alg->cipher_mode; 168 ctx_p->flow_mode = cc_alg->flow_mode; 169 ctx_p->drvdata = cc_alg->drvdata; 170 171 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 172 const char *name = crypto_tfm_alg_name(tfm); 173 174 /* Alloc hash tfm for essiv */ 175 ctx_p->shash_tfm = crypto_alloc_shash("sha256", 0, 0); 176 if (IS_ERR(ctx_p->shash_tfm)) { 177 dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); 178 return PTR_ERR(ctx_p->shash_tfm); 179 } 180 max_key_buf_size <<= 1; 181 182 /* Alloc fallabck tfm or essiv when key size != 256 bit */ 183 ctx_p->fallback_tfm = 184 crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); 185 186 if (IS_ERR(ctx_p->fallback_tfm)) { 187 /* Note we're still allowing registration with no fallback since it's 188 * better to have most modes supported than none at all. 189 */ 190 dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n", 191 name); 192 ctx_p->fallback_tfm = NULL; 193 } else { 194 fallback_req_size = crypto_skcipher_reqsize(ctx_p->fallback_tfm); 195 } 196 } 197 198 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 199 sizeof(struct cipher_req_ctx) + fallback_req_size); 200 201 /* Allocate key buffer, cache line aligned */ 202 ctx_p->user.key = kzalloc(max_key_buf_size, GFP_KERNEL); 203 if (!ctx_p->user.key) 204 goto free_fallback; 205 206 dev_dbg(dev, "Allocated key buffer in context. key=@%p\n", 207 ctx_p->user.key); 208 209 /* Map key buffer */ 210 ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key, 211 max_key_buf_size, 212 DMA_TO_DEVICE); 213 if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { 214 dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n", 215 max_key_buf_size, ctx_p->user.key); 216 goto free_key; 217 } 218 dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n", 219 max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); 220 221 return 0; 222 223free_key: 224 kfree(ctx_p->user.key); 225free_fallback: 226 crypto_free_skcipher(ctx_p->fallback_tfm); 227 crypto_free_shash(ctx_p->shash_tfm); 228 229 return -ENOMEM; 230} 231 232static void cc_cipher_exit(struct crypto_tfm *tfm) 233{ 234 struct crypto_alg *alg = tfm->__crt_alg; 235 struct cc_crypto_alg *cc_alg = 236 container_of(alg, struct cc_crypto_alg, 237 skcipher_alg.base); 238 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 239 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 240 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 241 242 dev_dbg(dev, "Clearing context @%p for %s\n", 243 crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm)); 244 245 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 246 /* Free hash tfm for essiv */ 247 crypto_free_shash(ctx_p->shash_tfm); 248 ctx_p->shash_tfm = NULL; 249 crypto_free_skcipher(ctx_p->fallback_tfm); 250 ctx_p->fallback_tfm = NULL; 251 } 252 253 /* Unmap key buffer */ 254 dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size, 255 DMA_TO_DEVICE); 256 dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n", 257 &ctx_p->user.key_dma_addr); 258 259 /* Free key buffer in context */ 260 dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key); 261 kfree_sensitive(ctx_p->user.key); 262} 263 264struct tdes_keys { 265 u8 key1[DES_KEY_SIZE]; 266 u8 key2[DES_KEY_SIZE]; 267 u8 key3[DES_KEY_SIZE]; 268}; 269 270static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num) 271{ 272 switch (slot_num) { 273 case 0: 274 return KFDE0_KEY; 275 case 1: 276 return KFDE1_KEY; 277 case 2: 278 return KFDE2_KEY; 279 case 3: 280 return KFDE3_KEY; 281 } 282 return END_OF_KEYS; 283} 284 285static u8 cc_slot_to_cpp_key(u8 slot_num) 286{ 287 return (slot_num - CC_FIRST_CPP_KEY_SLOT); 288} 289 290static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num) 291{ 292 if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT) 293 return CC_HW_PROTECTED_KEY; 294 else if (slot_num >= CC_FIRST_CPP_KEY_SLOT && 295 slot_num <= CC_LAST_CPP_KEY_SLOT) 296 return CC_POLICY_PROTECTED_KEY; 297 else 298 return CC_INVALID_PROTECTED_KEY; 299} 300 301static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, 302 unsigned int keylen) 303{ 304 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 305 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 306 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 307 struct cc_hkey_info hki; 308 309 dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n", 310 ctx_p, crypto_tfm_alg_name(tfm), keylen); 311 dump_byte_array("key", key, keylen); 312 313 /* STAT_PHASE_0: Init and sanity checks */ 314 315 /* This check the size of the protected key token */ 316 if (keylen != sizeof(hki)) { 317 dev_err(dev, "Unsupported protected key size %d.\n", keylen); 318 return -EINVAL; 319 } 320 321 memcpy(&hki, key, keylen); 322 323 /* The real key len for crypto op is the size of the HW key 324 * referenced by the HW key slot, not the hardware key token 325 */ 326 keylen = hki.keylen; 327 328 if (validate_keys_sizes(ctx_p, keylen)) { 329 dev_dbg(dev, "Unsupported key size %d.\n", keylen); 330 return -EINVAL; 331 } 332 333 ctx_p->keylen = keylen; 334 ctx_p->fallback_on = false; 335 336 switch (cc_slot_to_key_type(hki.hw_key1)) { 337 case CC_HW_PROTECTED_KEY: 338 if (ctx_p->flow_mode == S_DIN_to_SM4) { 339 dev_err(dev, "Only AES HW protected keys are supported\n"); 340 return -EINVAL; 341 } 342 343 ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); 344 if (ctx_p->hw.key1_slot == END_OF_KEYS) { 345 dev_err(dev, "Unsupported hw key1 number (%d)\n", 346 hki.hw_key1); 347 return -EINVAL; 348 } 349 350 if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 351 ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 352 if (hki.hw_key1 == hki.hw_key2) { 353 dev_err(dev, "Illegal hw key numbers (%d,%d)\n", 354 hki.hw_key1, hki.hw_key2); 355 return -EINVAL; 356 } 357 358 ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); 359 if (ctx_p->hw.key2_slot == END_OF_KEYS) { 360 dev_err(dev, "Unsupported hw key2 number (%d)\n", 361 hki.hw_key2); 362 return -EINVAL; 363 } 364 } 365 366 ctx_p->key_type = CC_HW_PROTECTED_KEY; 367 dev_dbg(dev, "HW protected key %d/%d set\n.", 368 ctx_p->hw.key1_slot, ctx_p->hw.key2_slot); 369 break; 370 371 case CC_POLICY_PROTECTED_KEY: 372 if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) { 373 dev_err(dev, "CPP keys not supported in this hardware revision.\n"); 374 return -EINVAL; 375 } 376 377 if (ctx_p->cipher_mode != DRV_CIPHER_CBC && 378 ctx_p->cipher_mode != DRV_CIPHER_CTR) { 379 dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n"); 380 return -EINVAL; 381 } 382 383 ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1); 384 if (ctx_p->flow_mode == S_DIN_to_AES) 385 ctx_p->cpp.alg = CC_CPP_AES; 386 else /* Must be SM4 since due to sethkey registration */ 387 ctx_p->cpp.alg = CC_CPP_SM4; 388 ctx_p->key_type = CC_POLICY_PROTECTED_KEY; 389 dev_dbg(dev, "policy protected key alg: %d slot: %d.\n", 390 ctx_p->cpp.alg, ctx_p->cpp.slot); 391 break; 392 393 default: 394 dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1); 395 return -EINVAL; 396 } 397 398 return 0; 399} 400 401static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, 402 unsigned int keylen) 403{ 404 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 405 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 406 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 407 struct cc_crypto_alg *cc_alg = 408 container_of(tfm->__crt_alg, struct cc_crypto_alg, 409 skcipher_alg.base); 410 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 411 412 dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n", 413 ctx_p, crypto_tfm_alg_name(tfm), keylen); 414 dump_byte_array("key", key, keylen); 415 416 /* STAT_PHASE_0: Init and sanity checks */ 417 418 if (validate_keys_sizes(ctx_p, keylen)) { 419 dev_dbg(dev, "Invalid key size %d.\n", keylen); 420 return -EINVAL; 421 } 422 423 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 424 425 /* We only support 256 bit ESSIV-CBC-AES keys */ 426 if (keylen != AES_KEYSIZE_256) { 427 unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK; 428 429 if (likely(ctx_p->fallback_tfm)) { 430 ctx_p->fallback_on = true; 431 crypto_skcipher_clear_flags(ctx_p->fallback_tfm, 432 CRYPTO_TFM_REQ_MASK); 433 crypto_skcipher_clear_flags(ctx_p->fallback_tfm, flags); 434 return crypto_skcipher_setkey(ctx_p->fallback_tfm, key, keylen); 435 } 436 437 dev_dbg(dev, "Unsupported key size %d and no fallback.\n", keylen); 438 return -EINVAL; 439 } 440 441 /* Internal ESSIV key buffer is double sized */ 442 max_key_buf_size <<= 1; 443 } 444 445 ctx_p->fallback_on = false; 446 ctx_p->key_type = CC_UNPROTECTED_KEY; 447 448 /* 449 * Verify DES weak keys 450 * Note that we're dropping the expanded key since the 451 * HW does the expansion on its own. 452 */ 453 if (ctx_p->flow_mode == S_DIN_to_DES) { 454 if ((keylen == DES3_EDE_KEY_SIZE && 455 verify_skcipher_des3_key(sktfm, key)) || 456 verify_skcipher_des_key(sktfm, key)) { 457 dev_dbg(dev, "weak DES key"); 458 return -EINVAL; 459 } 460 } 461 462 if (ctx_p->cipher_mode == DRV_CIPHER_XTS && 463 xts_verify_key(sktfm, key, keylen)) { 464 dev_dbg(dev, "weak XTS key"); 465 return -EINVAL; 466 } 467 468 /* STAT_PHASE_1: Copy key to ctx */ 469 dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr, 470 max_key_buf_size, DMA_TO_DEVICE); 471 472 memcpy(ctx_p->user.key, key, keylen); 473 474 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 475 /* sha256 for key2 - use sw implementation */ 476 int err; 477 478 err = crypto_shash_tfm_digest(ctx_p->shash_tfm, 479 ctx_p->user.key, keylen, 480 ctx_p->user.key + keylen); 481 if (err) { 482 dev_err(dev, "Failed to hash ESSIV key.\n"); 483 return err; 484 } 485 486 keylen <<= 1; 487 } 488 dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr, 489 max_key_buf_size, DMA_TO_DEVICE); 490 ctx_p->keylen = keylen; 491 492 dev_dbg(dev, "return safely"); 493 return 0; 494} 495 496static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p) 497{ 498 switch (ctx_p->flow_mode) { 499 case S_DIN_to_AES: 500 return S_AES_to_DOUT; 501 case S_DIN_to_DES: 502 return S_DES_to_DOUT; 503 case S_DIN_to_SM4: 504 return S_SM4_to_DOUT; 505 default: 506 return ctx_p->flow_mode; 507 } 508} 509 510static void cc_setup_readiv_desc(struct crypto_tfm *tfm, 511 struct cipher_req_ctx *req_ctx, 512 unsigned int ivsize, struct cc_hw_desc desc[], 513 unsigned int *seq_size) 514{ 515 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 516 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 517 int cipher_mode = ctx_p->cipher_mode; 518 int flow_mode = cc_out_setup_mode(ctx_p); 519 int direction = req_ctx->gen_ctx.op_type; 520 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 521 522 if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) 523 return; 524 525 switch (cipher_mode) { 526 case DRV_CIPHER_ECB: 527 break; 528 case DRV_CIPHER_CBC: 529 case DRV_CIPHER_CBC_CTS: 530 case DRV_CIPHER_CTR: 531 case DRV_CIPHER_OFB: 532 /* Read next IV */ 533 hw_desc_init(&desc[*seq_size]); 534 set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1); 535 set_cipher_config0(&desc[*seq_size], direction); 536 set_flow_mode(&desc[*seq_size], flow_mode); 537 set_cipher_mode(&desc[*seq_size], cipher_mode); 538 if (cipher_mode == DRV_CIPHER_CTR || 539 cipher_mode == DRV_CIPHER_OFB) { 540 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); 541 } else { 542 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0); 543 } 544 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 545 (*seq_size)++; 546 break; 547 case DRV_CIPHER_XTS: 548 case DRV_CIPHER_ESSIV: 549 /* IV */ 550 hw_desc_init(&desc[*seq_size]); 551 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); 552 set_cipher_mode(&desc[*seq_size], cipher_mode); 553 set_cipher_config0(&desc[*seq_size], direction); 554 set_flow_mode(&desc[*seq_size], flow_mode); 555 set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE, 556 NS_BIT, 1); 557 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 558 (*seq_size)++; 559 break; 560 default: 561 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 562 } 563} 564 565 566static void cc_setup_state_desc(struct crypto_tfm *tfm, 567 struct cipher_req_ctx *req_ctx, 568 unsigned int ivsize, unsigned int nbytes, 569 struct cc_hw_desc desc[], 570 unsigned int *seq_size) 571{ 572 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 573 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 574 int cipher_mode = ctx_p->cipher_mode; 575 int flow_mode = ctx_p->flow_mode; 576 int direction = req_ctx->gen_ctx.op_type; 577 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 578 579 switch (cipher_mode) { 580 case DRV_CIPHER_ECB: 581 break; 582 case DRV_CIPHER_CBC: 583 case DRV_CIPHER_CBC_CTS: 584 case DRV_CIPHER_CTR: 585 case DRV_CIPHER_OFB: 586 /* Load IV */ 587 hw_desc_init(&desc[*seq_size]); 588 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize, 589 NS_BIT); 590 set_cipher_config0(&desc[*seq_size], direction); 591 set_flow_mode(&desc[*seq_size], flow_mode); 592 set_cipher_mode(&desc[*seq_size], cipher_mode); 593 if (cipher_mode == DRV_CIPHER_CTR || 594 cipher_mode == DRV_CIPHER_OFB) { 595 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 596 } else { 597 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0); 598 } 599 (*seq_size)++; 600 break; 601 case DRV_CIPHER_XTS: 602 case DRV_CIPHER_ESSIV: 603 break; 604 default: 605 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 606 } 607} 608 609 610static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, 611 struct cipher_req_ctx *req_ctx, 612 unsigned int ivsize, unsigned int nbytes, 613 struct cc_hw_desc desc[], 614 unsigned int *seq_size) 615{ 616 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 617 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 618 int cipher_mode = ctx_p->cipher_mode; 619 int flow_mode = ctx_p->flow_mode; 620 int direction = req_ctx->gen_ctx.op_type; 621 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 622 unsigned int key_len = (ctx_p->keylen / 2); 623 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 624 unsigned int key_offset = key_len; 625 626 switch (cipher_mode) { 627 case DRV_CIPHER_ECB: 628 break; 629 case DRV_CIPHER_CBC: 630 case DRV_CIPHER_CBC_CTS: 631 case DRV_CIPHER_CTR: 632 case DRV_CIPHER_OFB: 633 break; 634 case DRV_CIPHER_XTS: 635 case DRV_CIPHER_ESSIV: 636 637 if (cipher_mode == DRV_CIPHER_ESSIV) 638 key_len = SHA256_DIGEST_SIZE; 639 640 /* load XEX key */ 641 hw_desc_init(&desc[*seq_size]); 642 set_cipher_mode(&desc[*seq_size], cipher_mode); 643 set_cipher_config0(&desc[*seq_size], direction); 644 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 645 set_hw_crypto_key(&desc[*seq_size], 646 ctx_p->hw.key2_slot); 647 } else { 648 set_din_type(&desc[*seq_size], DMA_DLLI, 649 (key_dma_addr + key_offset), 650 key_len, NS_BIT); 651 } 652 set_xex_data_unit_size(&desc[*seq_size], nbytes); 653 set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); 654 set_key_size_aes(&desc[*seq_size], key_len); 655 set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); 656 (*seq_size)++; 657 658 /* Load IV */ 659 hw_desc_init(&desc[*seq_size]); 660 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 661 set_cipher_mode(&desc[*seq_size], cipher_mode); 662 set_cipher_config0(&desc[*seq_size], direction); 663 set_key_size_aes(&desc[*seq_size], key_len); 664 set_flow_mode(&desc[*seq_size], flow_mode); 665 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, 666 CC_AES_BLOCK_SIZE, NS_BIT); 667 (*seq_size)++; 668 break; 669 default: 670 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 671 } 672} 673 674static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p) 675{ 676 switch (ctx_p->flow_mode) { 677 case S_DIN_to_AES: 678 return DIN_AES_DOUT; 679 case S_DIN_to_DES: 680 return DIN_DES_DOUT; 681 case S_DIN_to_SM4: 682 return DIN_SM4_DOUT; 683 default: 684 return ctx_p->flow_mode; 685 } 686} 687 688static void cc_setup_key_desc(struct crypto_tfm *tfm, 689 struct cipher_req_ctx *req_ctx, 690 unsigned int nbytes, struct cc_hw_desc desc[], 691 unsigned int *seq_size) 692{ 693 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 694 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 695 int cipher_mode = ctx_p->cipher_mode; 696 int flow_mode = ctx_p->flow_mode; 697 int direction = req_ctx->gen_ctx.op_type; 698 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 699 unsigned int key_len = ctx_p->keylen; 700 unsigned int din_size; 701 702 switch (cipher_mode) { 703 case DRV_CIPHER_CBC: 704 case DRV_CIPHER_CBC_CTS: 705 case DRV_CIPHER_CTR: 706 case DRV_CIPHER_OFB: 707 case DRV_CIPHER_ECB: 708 /* Load key */ 709 hw_desc_init(&desc[*seq_size]); 710 set_cipher_mode(&desc[*seq_size], cipher_mode); 711 set_cipher_config0(&desc[*seq_size], direction); 712 713 if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) { 714 /* We use the AES key size coding for all CPP algs */ 715 set_key_size_aes(&desc[*seq_size], key_len); 716 set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot); 717 flow_mode = cc_out_flow_mode(ctx_p); 718 } else { 719 if (flow_mode == S_DIN_to_AES) { 720 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 721 set_hw_crypto_key(&desc[*seq_size], 722 ctx_p->hw.key1_slot); 723 } else { 724 /* CC_POLICY_UNPROTECTED_KEY 725 * Invalid keys are filtered out in 726 * sethkey() 727 */ 728 din_size = (key_len == 24) ? 729 AES_MAX_KEY_SIZE : key_len; 730 731 set_din_type(&desc[*seq_size], DMA_DLLI, 732 key_dma_addr, din_size, 733 NS_BIT); 734 } 735 set_key_size_aes(&desc[*seq_size], key_len); 736 } else { 737 /*des*/ 738 set_din_type(&desc[*seq_size], DMA_DLLI, 739 key_dma_addr, key_len, NS_BIT); 740 set_key_size_des(&desc[*seq_size], key_len); 741 } 742 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 743 } 744 set_flow_mode(&desc[*seq_size], flow_mode); 745 (*seq_size)++; 746 break; 747 case DRV_CIPHER_XTS: 748 case DRV_CIPHER_ESSIV: 749 /* Load AES key */ 750 hw_desc_init(&desc[*seq_size]); 751 set_cipher_mode(&desc[*seq_size], cipher_mode); 752 set_cipher_config0(&desc[*seq_size], direction); 753 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 754 set_hw_crypto_key(&desc[*seq_size], 755 ctx_p->hw.key1_slot); 756 } else { 757 set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, 758 (key_len / 2), NS_BIT); 759 } 760 set_key_size_aes(&desc[*seq_size], (key_len / 2)); 761 set_flow_mode(&desc[*seq_size], flow_mode); 762 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 763 (*seq_size)++; 764 break; 765 default: 766 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 767 } 768} 769 770static void cc_setup_mlli_desc(struct crypto_tfm *tfm, 771 struct cipher_req_ctx *req_ctx, 772 struct scatterlist *dst, struct scatterlist *src, 773 unsigned int nbytes, void *areq, 774 struct cc_hw_desc desc[], unsigned int *seq_size) 775{ 776 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 777 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 778 779 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { 780 /* bypass */ 781 dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n", 782 &req_ctx->mlli_params.mlli_dma_addr, 783 req_ctx->mlli_params.mlli_len, 784 ctx_p->drvdata->mlli_sram_addr); 785 hw_desc_init(&desc[*seq_size]); 786 set_din_type(&desc[*seq_size], DMA_DLLI, 787 req_ctx->mlli_params.mlli_dma_addr, 788 req_ctx->mlli_params.mlli_len, NS_BIT); 789 set_dout_sram(&desc[*seq_size], 790 ctx_p->drvdata->mlli_sram_addr, 791 req_ctx->mlli_params.mlli_len); 792 set_flow_mode(&desc[*seq_size], BYPASS); 793 (*seq_size)++; 794 } 795} 796 797static void cc_setup_flow_desc(struct crypto_tfm *tfm, 798 struct cipher_req_ctx *req_ctx, 799 struct scatterlist *dst, struct scatterlist *src, 800 unsigned int nbytes, struct cc_hw_desc desc[], 801 unsigned int *seq_size) 802{ 803 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 804 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 805 unsigned int flow_mode = cc_out_flow_mode(ctx_p); 806 bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY || 807 ctx_p->cipher_mode == DRV_CIPHER_ECB); 808 809 /* Process */ 810 if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { 811 dev_dbg(dev, " data params addr %pad length 0x%X\n", 812 &sg_dma_address(src), nbytes); 813 dev_dbg(dev, " data params addr %pad length 0x%X\n", 814 &sg_dma_address(dst), nbytes); 815 hw_desc_init(&desc[*seq_size]); 816 set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), 817 nbytes, NS_BIT); 818 set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), 819 nbytes, NS_BIT, (!last_desc ? 0 : 1)); 820 if (last_desc) 821 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 822 823 set_flow_mode(&desc[*seq_size], flow_mode); 824 (*seq_size)++; 825 } else { 826 hw_desc_init(&desc[*seq_size]); 827 set_din_type(&desc[*seq_size], DMA_MLLI, 828 ctx_p->drvdata->mlli_sram_addr, 829 req_ctx->in_mlli_nents, NS_BIT); 830 if (req_ctx->out_nents == 0) { 831 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 832 ctx_p->drvdata->mlli_sram_addr, 833 ctx_p->drvdata->mlli_sram_addr); 834 set_dout_mlli(&desc[*seq_size], 835 ctx_p->drvdata->mlli_sram_addr, 836 req_ctx->in_mlli_nents, NS_BIT, 837 (!last_desc ? 0 : 1)); 838 } else { 839 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 840 ctx_p->drvdata->mlli_sram_addr, 841 ctx_p->drvdata->mlli_sram_addr + 842 (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents); 843 set_dout_mlli(&desc[*seq_size], 844 (ctx_p->drvdata->mlli_sram_addr + 845 (LLI_ENTRY_BYTE_SIZE * 846 req_ctx->in_mlli_nents)), 847 req_ctx->out_mlli_nents, NS_BIT, 848 (!last_desc ? 0 : 1)); 849 } 850 if (last_desc) 851 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 852 853 set_flow_mode(&desc[*seq_size], flow_mode); 854 (*seq_size)++; 855 } 856} 857 858static void cc_cipher_complete(struct device *dev, void *cc_req, int err) 859{ 860 struct skcipher_request *req = (struct skcipher_request *)cc_req; 861 struct scatterlist *dst = req->dst; 862 struct scatterlist *src = req->src; 863 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 864 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 865 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 866 867 if (err != -EINPROGRESS) { 868 /* Not a BACKLOG notification */ 869 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 870 memcpy(req->iv, req_ctx->iv, ivsize); 871 kfree_sensitive(req_ctx->iv); 872 } 873 874 skcipher_request_complete(req, err); 875} 876 877static int cc_cipher_process(struct skcipher_request *req, 878 enum drv_crypto_direction direction) 879{ 880 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 881 struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); 882 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 883 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 884 struct scatterlist *dst = req->dst; 885 struct scatterlist *src = req->src; 886 unsigned int nbytes = req->cryptlen; 887 void *iv = req->iv; 888 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 889 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 890 struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN]; 891 struct cc_crypto_req cc_req = {}; 892 int rc; 893 unsigned int seq_len = 0; 894 gfp_t flags = cc_gfp_flags(&req->base); 895 896 dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n", 897 ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 898 "Encrypt" : "Decrypt"), req, iv, nbytes); 899 900 /* STAT_PHASE_0: Init and sanity checks */ 901 902 if (validate_data_size(ctx_p, nbytes)) { 903 dev_dbg(dev, "Unsupported data size %d.\n", nbytes); 904 rc = -EINVAL; 905 goto exit_process; 906 } 907 if (nbytes == 0) { 908 /* No data to process is valid */ 909 rc = 0; 910 goto exit_process; 911 } 912 913 if (ctx_p->fallback_on) { 914 struct skcipher_request *subreq = skcipher_request_ctx(req); 915 916 *subreq = *req; 917 skcipher_request_set_tfm(subreq, ctx_p->fallback_tfm); 918 if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT) 919 return crypto_skcipher_encrypt(subreq); 920 else 921 return crypto_skcipher_decrypt(subreq); 922 } 923 924 /* The IV we are handed may be allocated from the stack so 925 * we must copy it to a DMAable buffer before use. 926 */ 927 req_ctx->iv = kmemdup(iv, ivsize, flags); 928 if (!req_ctx->iv) { 929 rc = -ENOMEM; 930 goto exit_process; 931 } 932 933 /* Setup request structure */ 934 cc_req.user_cb = cc_cipher_complete; 935 cc_req.user_arg = req; 936 937 /* Setup CPP operation details */ 938 if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) { 939 cc_req.cpp.is_cpp = true; 940 cc_req.cpp.alg = ctx_p->cpp.alg; 941 cc_req.cpp.slot = ctx_p->cpp.slot; 942 } 943 944 /* Setup request context */ 945 req_ctx->gen_ctx.op_type = direction; 946 947 /* STAT_PHASE_1: Map buffers */ 948 949 rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes, 950 req_ctx->iv, src, dst, flags); 951 if (rc) { 952 dev_err(dev, "map_request() failed\n"); 953 goto exit_process; 954 } 955 956 /* STAT_PHASE_2: Create sequence */ 957 958 /* Setup state (IV) */ 959 cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 960 /* Setup MLLI line, if needed */ 961 cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len); 962 /* Setup key */ 963 cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len); 964 /* Setup state (IV and XEX key) */ 965 cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 966 /* Data processing */ 967 cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len); 968 /* Read next IV */ 969 cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len); 970 971 /* STAT_PHASE_3: Lock HW and push sequence */ 972 973 rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len, 974 &req->base); 975 if (rc != -EINPROGRESS && rc != -EBUSY) { 976 /* Failed to send the request or request completed 977 * synchronously 978 */ 979 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 980 } 981 982exit_process: 983 if (rc != -EINPROGRESS && rc != -EBUSY) { 984 kfree_sensitive(req_ctx->iv); 985 } 986 987 return rc; 988} 989 990static int cc_cipher_encrypt(struct skcipher_request *req) 991{ 992 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 993 994 memset(req_ctx, 0, sizeof(*req_ctx)); 995 996 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 997} 998 999static int cc_cipher_decrypt(struct skcipher_request *req) 1000{ 1001 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 1002 1003 memset(req_ctx, 0, sizeof(*req_ctx)); 1004 1005 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT); 1006} 1007 1008/* Block cipher alg */ 1009static const struct cc_alg_template skcipher_algs[] = { 1010 { 1011 .name = "xts(paes)", 1012 .driver_name = "xts-paes-ccree", 1013 .blocksize = 1, 1014 .template_skcipher = { 1015 .setkey = cc_cipher_sethkey, 1016 .encrypt = cc_cipher_encrypt, 1017 .decrypt = cc_cipher_decrypt, 1018 .min_keysize = CC_HW_KEY_SIZE, 1019 .max_keysize = CC_HW_KEY_SIZE, 1020 .ivsize = AES_BLOCK_SIZE, 1021 }, 1022 .cipher_mode = DRV_CIPHER_XTS, 1023 .flow_mode = S_DIN_to_AES, 1024 .min_hw_rev = CC_HW_REV_630, 1025 .std_body = CC_STD_NIST, 1026 .sec_func = true, 1027 }, 1028 { 1029 .name = "essiv(cbc(paes),sha256)", 1030 .driver_name = "essiv-paes-ccree", 1031 .blocksize = AES_BLOCK_SIZE, 1032 .template_skcipher = { 1033 .setkey = cc_cipher_sethkey, 1034 .encrypt = cc_cipher_encrypt, 1035 .decrypt = cc_cipher_decrypt, 1036 .min_keysize = CC_HW_KEY_SIZE, 1037 .max_keysize = CC_HW_KEY_SIZE, 1038 .ivsize = AES_BLOCK_SIZE, 1039 }, 1040 .cipher_mode = DRV_CIPHER_ESSIV, 1041 .flow_mode = S_DIN_to_AES, 1042 .min_hw_rev = CC_HW_REV_712, 1043 .std_body = CC_STD_NIST, 1044 .sec_func = true, 1045 }, 1046 { 1047 .name = "ecb(paes)", 1048 .driver_name = "ecb-paes-ccree", 1049 .blocksize = AES_BLOCK_SIZE, 1050 .template_skcipher = { 1051 .setkey = cc_cipher_sethkey, 1052 .encrypt = cc_cipher_encrypt, 1053 .decrypt = cc_cipher_decrypt, 1054 .min_keysize = CC_HW_KEY_SIZE, 1055 .max_keysize = CC_HW_KEY_SIZE, 1056 .ivsize = 0, 1057 }, 1058 .cipher_mode = DRV_CIPHER_ECB, 1059 .flow_mode = S_DIN_to_AES, 1060 .min_hw_rev = CC_HW_REV_712, 1061 .std_body = CC_STD_NIST, 1062 .sec_func = true, 1063 }, 1064 { 1065 .name = "cbc(paes)", 1066 .driver_name = "cbc-paes-ccree", 1067 .blocksize = AES_BLOCK_SIZE, 1068 .template_skcipher = { 1069 .setkey = cc_cipher_sethkey, 1070 .encrypt = cc_cipher_encrypt, 1071 .decrypt = cc_cipher_decrypt, 1072 .min_keysize = CC_HW_KEY_SIZE, 1073 .max_keysize = CC_HW_KEY_SIZE, 1074 .ivsize = AES_BLOCK_SIZE, 1075 }, 1076 .cipher_mode = DRV_CIPHER_CBC, 1077 .flow_mode = S_DIN_to_AES, 1078 .min_hw_rev = CC_HW_REV_712, 1079 .std_body = CC_STD_NIST, 1080 .sec_func = true, 1081 }, 1082 { 1083 .name = "cts(cbc(paes))", 1084 .driver_name = "cts-cbc-paes-ccree", 1085 .blocksize = AES_BLOCK_SIZE, 1086 .template_skcipher = { 1087 .setkey = cc_cipher_sethkey, 1088 .encrypt = cc_cipher_encrypt, 1089 .decrypt = cc_cipher_decrypt, 1090 .min_keysize = CC_HW_KEY_SIZE, 1091 .max_keysize = CC_HW_KEY_SIZE, 1092 .ivsize = AES_BLOCK_SIZE, 1093 }, 1094 .cipher_mode = DRV_CIPHER_CBC_CTS, 1095 .flow_mode = S_DIN_to_AES, 1096 .min_hw_rev = CC_HW_REV_712, 1097 .std_body = CC_STD_NIST, 1098 .sec_func = true, 1099 }, 1100 { 1101 .name = "ctr(paes)", 1102 .driver_name = "ctr-paes-ccree", 1103 .blocksize = 1, 1104 .template_skcipher = { 1105 .setkey = cc_cipher_sethkey, 1106 .encrypt = cc_cipher_encrypt, 1107 .decrypt = cc_cipher_decrypt, 1108 .min_keysize = CC_HW_KEY_SIZE, 1109 .max_keysize = CC_HW_KEY_SIZE, 1110 .ivsize = AES_BLOCK_SIZE, 1111 }, 1112 .cipher_mode = DRV_CIPHER_CTR, 1113 .flow_mode = S_DIN_to_AES, 1114 .min_hw_rev = CC_HW_REV_712, 1115 .std_body = CC_STD_NIST, 1116 .sec_func = true, 1117 }, 1118 { 1119 /* See https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg40576.html 1120 * for the reason why this differs from the generic 1121 * implementation. 1122 */ 1123 .name = "xts(aes)", 1124 .driver_name = "xts-aes-ccree", 1125 .blocksize = 1, 1126 .template_skcipher = { 1127 .setkey = cc_cipher_setkey, 1128 .encrypt = cc_cipher_encrypt, 1129 .decrypt = cc_cipher_decrypt, 1130 .min_keysize = AES_MIN_KEY_SIZE * 2, 1131 .max_keysize = AES_MAX_KEY_SIZE * 2, 1132 .ivsize = AES_BLOCK_SIZE, 1133 }, 1134 .cipher_mode = DRV_CIPHER_XTS, 1135 .flow_mode = S_DIN_to_AES, 1136 .min_hw_rev = CC_HW_REV_630, 1137 .std_body = CC_STD_NIST, 1138 }, 1139 { 1140 .name = "essiv(cbc(aes),sha256)", 1141 .driver_name = "essiv-aes-ccree", 1142 .blocksize = AES_BLOCK_SIZE, 1143 .template_skcipher = { 1144 .setkey = cc_cipher_setkey, 1145 .encrypt = cc_cipher_encrypt, 1146 .decrypt = cc_cipher_decrypt, 1147 .min_keysize = AES_MIN_KEY_SIZE, 1148 .max_keysize = AES_MAX_KEY_SIZE, 1149 .ivsize = AES_BLOCK_SIZE, 1150 }, 1151 .cipher_mode = DRV_CIPHER_ESSIV, 1152 .flow_mode = S_DIN_to_AES, 1153 .min_hw_rev = CC_HW_REV_712, 1154 .std_body = CC_STD_NIST, 1155 }, 1156 { 1157 .name = "ecb(aes)", 1158 .driver_name = "ecb-aes-ccree", 1159 .blocksize = AES_BLOCK_SIZE, 1160 .template_skcipher = { 1161 .setkey = cc_cipher_setkey, 1162 .encrypt = cc_cipher_encrypt, 1163 .decrypt = cc_cipher_decrypt, 1164 .min_keysize = AES_MIN_KEY_SIZE, 1165 .max_keysize = AES_MAX_KEY_SIZE, 1166 .ivsize = 0, 1167 }, 1168 .cipher_mode = DRV_CIPHER_ECB, 1169 .flow_mode = S_DIN_to_AES, 1170 .min_hw_rev = CC_HW_REV_630, 1171 .std_body = CC_STD_NIST, 1172 }, 1173 { 1174 .name = "cbc(aes)", 1175 .driver_name = "cbc-aes-ccree", 1176 .blocksize = AES_BLOCK_SIZE, 1177 .template_skcipher = { 1178 .setkey = cc_cipher_setkey, 1179 .encrypt = cc_cipher_encrypt, 1180 .decrypt = cc_cipher_decrypt, 1181 .min_keysize = AES_MIN_KEY_SIZE, 1182 .max_keysize = AES_MAX_KEY_SIZE, 1183 .ivsize = AES_BLOCK_SIZE, 1184 }, 1185 .cipher_mode = DRV_CIPHER_CBC, 1186 .flow_mode = S_DIN_to_AES, 1187 .min_hw_rev = CC_HW_REV_630, 1188 .std_body = CC_STD_NIST, 1189 }, 1190 { 1191 .name = "cts(cbc(aes))", 1192 .driver_name = "cts-cbc-aes-ccree", 1193 .blocksize = AES_BLOCK_SIZE, 1194 .template_skcipher = { 1195 .setkey = cc_cipher_setkey, 1196 .encrypt = cc_cipher_encrypt, 1197 .decrypt = cc_cipher_decrypt, 1198 .min_keysize = AES_MIN_KEY_SIZE, 1199 .max_keysize = AES_MAX_KEY_SIZE, 1200 .ivsize = AES_BLOCK_SIZE, 1201 }, 1202 .cipher_mode = DRV_CIPHER_CBC_CTS, 1203 .flow_mode = S_DIN_to_AES, 1204 .min_hw_rev = CC_HW_REV_630, 1205 .std_body = CC_STD_NIST, 1206 }, 1207 { 1208 .name = "ctr(aes)", 1209 .driver_name = "ctr-aes-ccree", 1210 .blocksize = 1, 1211 .template_skcipher = { 1212 .setkey = cc_cipher_setkey, 1213 .encrypt = cc_cipher_encrypt, 1214 .decrypt = cc_cipher_decrypt, 1215 .min_keysize = AES_MIN_KEY_SIZE, 1216 .max_keysize = AES_MAX_KEY_SIZE, 1217 .ivsize = AES_BLOCK_SIZE, 1218 }, 1219 .cipher_mode = DRV_CIPHER_CTR, 1220 .flow_mode = S_DIN_to_AES, 1221 .min_hw_rev = CC_HW_REV_630, 1222 .std_body = CC_STD_NIST, 1223 }, 1224 { 1225 .name = "cbc(des3_ede)", 1226 .driver_name = "cbc-3des-ccree", 1227 .blocksize = DES3_EDE_BLOCK_SIZE, 1228 .template_skcipher = { 1229 .setkey = cc_cipher_setkey, 1230 .encrypt = cc_cipher_encrypt, 1231 .decrypt = cc_cipher_decrypt, 1232 .min_keysize = DES3_EDE_KEY_SIZE, 1233 .max_keysize = DES3_EDE_KEY_SIZE, 1234 .ivsize = DES3_EDE_BLOCK_SIZE, 1235 }, 1236 .cipher_mode = DRV_CIPHER_CBC, 1237 .flow_mode = S_DIN_to_DES, 1238 .min_hw_rev = CC_HW_REV_630, 1239 .std_body = CC_STD_NIST, 1240 }, 1241 { 1242 .name = "ecb(des3_ede)", 1243 .driver_name = "ecb-3des-ccree", 1244 .blocksize = DES3_EDE_BLOCK_SIZE, 1245 .template_skcipher = { 1246 .setkey = cc_cipher_setkey, 1247 .encrypt = cc_cipher_encrypt, 1248 .decrypt = cc_cipher_decrypt, 1249 .min_keysize = DES3_EDE_KEY_SIZE, 1250 .max_keysize = DES3_EDE_KEY_SIZE, 1251 .ivsize = 0, 1252 }, 1253 .cipher_mode = DRV_CIPHER_ECB, 1254 .flow_mode = S_DIN_to_DES, 1255 .min_hw_rev = CC_HW_REV_630, 1256 .std_body = CC_STD_NIST, 1257 }, 1258 { 1259 .name = "cbc(des)", 1260 .driver_name = "cbc-des-ccree", 1261 .blocksize = DES_BLOCK_SIZE, 1262 .template_skcipher = { 1263 .setkey = cc_cipher_setkey, 1264 .encrypt = cc_cipher_encrypt, 1265 .decrypt = cc_cipher_decrypt, 1266 .min_keysize = DES_KEY_SIZE, 1267 .max_keysize = DES_KEY_SIZE, 1268 .ivsize = DES_BLOCK_SIZE, 1269 }, 1270 .cipher_mode = DRV_CIPHER_CBC, 1271 .flow_mode = S_DIN_to_DES, 1272 .min_hw_rev = CC_HW_REV_630, 1273 .std_body = CC_STD_NIST, 1274 }, 1275 { 1276 .name = "ecb(des)", 1277 .driver_name = "ecb-des-ccree", 1278 .blocksize = DES_BLOCK_SIZE, 1279 .template_skcipher = { 1280 .setkey = cc_cipher_setkey, 1281 .encrypt = cc_cipher_encrypt, 1282 .decrypt = cc_cipher_decrypt, 1283 .min_keysize = DES_KEY_SIZE, 1284 .max_keysize = DES_KEY_SIZE, 1285 .ivsize = 0, 1286 }, 1287 .cipher_mode = DRV_CIPHER_ECB, 1288 .flow_mode = S_DIN_to_DES, 1289 .min_hw_rev = CC_HW_REV_630, 1290 .std_body = CC_STD_NIST, 1291 }, 1292 { 1293 .name = "cbc(sm4)", 1294 .driver_name = "cbc-sm4-ccree", 1295 .blocksize = SM4_BLOCK_SIZE, 1296 .template_skcipher = { 1297 .setkey = cc_cipher_setkey, 1298 .encrypt = cc_cipher_encrypt, 1299 .decrypt = cc_cipher_decrypt, 1300 .min_keysize = SM4_KEY_SIZE, 1301 .max_keysize = SM4_KEY_SIZE, 1302 .ivsize = SM4_BLOCK_SIZE, 1303 }, 1304 .cipher_mode = DRV_CIPHER_CBC, 1305 .flow_mode = S_DIN_to_SM4, 1306 .min_hw_rev = CC_HW_REV_713, 1307 .std_body = CC_STD_OSCCA, 1308 }, 1309 { 1310 .name = "ecb(sm4)", 1311 .driver_name = "ecb-sm4-ccree", 1312 .blocksize = SM4_BLOCK_SIZE, 1313 .template_skcipher = { 1314 .setkey = cc_cipher_setkey, 1315 .encrypt = cc_cipher_encrypt, 1316 .decrypt = cc_cipher_decrypt, 1317 .min_keysize = SM4_KEY_SIZE, 1318 .max_keysize = SM4_KEY_SIZE, 1319 .ivsize = 0, 1320 }, 1321 .cipher_mode = DRV_CIPHER_ECB, 1322 .flow_mode = S_DIN_to_SM4, 1323 .min_hw_rev = CC_HW_REV_713, 1324 .std_body = CC_STD_OSCCA, 1325 }, 1326 { 1327 .name = "ctr(sm4)", 1328 .driver_name = "ctr-sm4-ccree", 1329 .blocksize = 1, 1330 .template_skcipher = { 1331 .setkey = cc_cipher_setkey, 1332 .encrypt = cc_cipher_encrypt, 1333 .decrypt = cc_cipher_decrypt, 1334 .min_keysize = SM4_KEY_SIZE, 1335 .max_keysize = SM4_KEY_SIZE, 1336 .ivsize = SM4_BLOCK_SIZE, 1337 }, 1338 .cipher_mode = DRV_CIPHER_CTR, 1339 .flow_mode = S_DIN_to_SM4, 1340 .min_hw_rev = CC_HW_REV_713, 1341 .std_body = CC_STD_OSCCA, 1342 }, 1343 { 1344 .name = "cbc(psm4)", 1345 .driver_name = "cbc-psm4-ccree", 1346 .blocksize = SM4_BLOCK_SIZE, 1347 .template_skcipher = { 1348 .setkey = cc_cipher_sethkey, 1349 .encrypt = cc_cipher_encrypt, 1350 .decrypt = cc_cipher_decrypt, 1351 .min_keysize = CC_HW_KEY_SIZE, 1352 .max_keysize = CC_HW_KEY_SIZE, 1353 .ivsize = SM4_BLOCK_SIZE, 1354 }, 1355 .cipher_mode = DRV_CIPHER_CBC, 1356 .flow_mode = S_DIN_to_SM4, 1357 .min_hw_rev = CC_HW_REV_713, 1358 .std_body = CC_STD_OSCCA, 1359 .sec_func = true, 1360 }, 1361 { 1362 .name = "ctr(psm4)", 1363 .driver_name = "ctr-psm4-ccree", 1364 .blocksize = SM4_BLOCK_SIZE, 1365 .template_skcipher = { 1366 .setkey = cc_cipher_sethkey, 1367 .encrypt = cc_cipher_encrypt, 1368 .decrypt = cc_cipher_decrypt, 1369 .min_keysize = CC_HW_KEY_SIZE, 1370 .max_keysize = CC_HW_KEY_SIZE, 1371 .ivsize = SM4_BLOCK_SIZE, 1372 }, 1373 .cipher_mode = DRV_CIPHER_CTR, 1374 .flow_mode = S_DIN_to_SM4, 1375 .min_hw_rev = CC_HW_REV_713, 1376 .std_body = CC_STD_OSCCA, 1377 .sec_func = true, 1378 }, 1379}; 1380 1381static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, 1382 struct device *dev) 1383{ 1384 struct cc_crypto_alg *t_alg; 1385 struct skcipher_alg *alg; 1386 1387 t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); 1388 if (!t_alg) 1389 return ERR_PTR(-ENOMEM); 1390 1391 alg = &t_alg->skcipher_alg; 1392 1393 memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); 1394 1395 if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1396 tmpl->name) >= CRYPTO_MAX_ALG_NAME) 1397 return ERR_PTR(-EINVAL); 1398 if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1399 tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME) 1400 return ERR_PTR(-EINVAL); 1401 1402 alg->base.cra_module = THIS_MODULE; 1403 alg->base.cra_priority = CC_CRA_PRIO; 1404 alg->base.cra_blocksize = tmpl->blocksize; 1405 alg->base.cra_alignmask = 0; 1406 alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx); 1407 1408 alg->base.cra_init = cc_cipher_init; 1409 alg->base.cra_exit = cc_cipher_exit; 1410 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 1411 1412 t_alg->cipher_mode = tmpl->cipher_mode; 1413 t_alg->flow_mode = tmpl->flow_mode; 1414 1415 return t_alg; 1416} 1417 1418int cc_cipher_free(struct cc_drvdata *drvdata) 1419{ 1420 struct cc_crypto_alg *t_alg, *n; 1421 1422 /* Remove registered algs */ 1423 list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) { 1424 crypto_unregister_skcipher(&t_alg->skcipher_alg); 1425 list_del(&t_alg->entry); 1426 } 1427 return 0; 1428} 1429 1430int cc_cipher_alloc(struct cc_drvdata *drvdata) 1431{ 1432 struct cc_crypto_alg *t_alg; 1433 struct device *dev = drvdata_to_dev(drvdata); 1434 int rc = -ENOMEM; 1435 int alg; 1436 1437 INIT_LIST_HEAD(&drvdata->alg_list); 1438 1439 /* Linux crypto */ 1440 dev_dbg(dev, "Number of algorithms = %zu\n", 1441 ARRAY_SIZE(skcipher_algs)); 1442 for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { 1443 if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) || 1444 !(drvdata->std_bodies & skcipher_algs[alg].std_body) || 1445 (drvdata->sec_disabled && skcipher_algs[alg].sec_func)) 1446 continue; 1447 1448 dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name); 1449 t_alg = cc_create_alg(&skcipher_algs[alg], dev); 1450 if (IS_ERR(t_alg)) { 1451 rc = PTR_ERR(t_alg); 1452 dev_err(dev, "%s alg allocation failed\n", 1453 skcipher_algs[alg].driver_name); 1454 goto fail0; 1455 } 1456 t_alg->drvdata = drvdata; 1457 1458 dev_dbg(dev, "registering %s\n", 1459 skcipher_algs[alg].driver_name); 1460 rc = crypto_register_skcipher(&t_alg->skcipher_alg); 1461 dev_dbg(dev, "%s alg registration rc = %x\n", 1462 t_alg->skcipher_alg.base.cra_driver_name, rc); 1463 if (rc) { 1464 dev_err(dev, "%s alg registration failed\n", 1465 t_alg->skcipher_alg.base.cra_driver_name); 1466 goto fail0; 1467 } 1468 1469 list_add_tail(&t_alg->entry, &drvdata->alg_list); 1470 dev_dbg(dev, "Registered %s\n", 1471 t_alg->skcipher_alg.base.cra_driver_name); 1472 } 1473 return 0; 1474 1475fail0: 1476 cc_cipher_free(drvdata); 1477 return rc; 1478} 1479