1// SPDX-License-Identifier: GPL-2.0 2/* 3 * sun8i-ce-core.c - hardware cryptographic offloader for 4 * Allwinner H3/A64/H5/H2+/H6/R40 SoC 5 * 6 * Copyright (C) 2015-2019 Corentin Labbe <clabbe.montjoie@gmail.com> 7 * 8 * Core file which registers crypto algorithms supported by the CryptoEngine. 9 * 10 * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst 11 */ 12 13#include <crypto/engine.h> 14#include <crypto/internal/hash.h> 15#include <crypto/internal/rng.h> 16#include <crypto/internal/skcipher.h> 17#include <linux/clk.h> 18#include <linux/delay.h> 19#include <linux/dma-mapping.h> 20#include <linux/err.h> 21#include <linux/interrupt.h> 22#include <linux/io.h> 23#include <linux/irq.h> 24#include <linux/kernel.h> 25#include <linux/module.h> 26#include <linux/of.h> 27#include <linux/platform_device.h> 28#include <linux/pm_runtime.h> 29#include <linux/reset.h> 30 31#include "sun8i-ce.h" 32 33/* 34 * mod clock is lower on H3 than other SoC due to some DMA timeout occurring 35 * with high value. 36 * If you want to tune mod clock, loading driver and passing selftest is 37 * insufficient, you need to test with some LUKS test (mount and write to it) 38 */ 39static const struct ce_variant ce_h3_variant = { 40 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 41 }, 42 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 43 CE_ALG_SHA384, CE_ALG_SHA512 44 }, 45 .op_mode = { CE_OP_ECB, CE_OP_CBC 46 }, 47 .ce_clks = { 48 { "bus", 0, 200000000 }, 49 { "mod", 50000000, 0 }, 50 }, 51 .esr = ESR_H3, 52 .prng = CE_ALG_PRNG, 53 .trng = CE_ID_NOTSUPP, 54}; 55 56static const struct ce_variant ce_h5_variant = { 57 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 58 }, 59 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 60 CE_ID_NOTSUPP, CE_ID_NOTSUPP 61 }, 62 .op_mode = { CE_OP_ECB, CE_OP_CBC 63 }, 64 .ce_clks = { 65 { "bus", 0, 200000000 }, 66 { "mod", 300000000, 0 }, 67 }, 68 .esr = ESR_H5, 69 .prng = CE_ALG_PRNG, 70 .trng = CE_ID_NOTSUPP, 71}; 72 73static const struct ce_variant ce_h6_variant = { 74 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 75 }, 76 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 77 CE_ALG_SHA384, CE_ALG_SHA512 78 }, 79 .op_mode = { CE_OP_ECB, CE_OP_CBC 80 }, 81 .cipher_t_dlen_in_bytes = true, 82 .hash_t_dlen_in_bits = true, 83 .prng_t_dlen_in_bytes = true, 84 .trng_t_dlen_in_bytes = true, 85 .ce_clks = { 86 { "bus", 0, 200000000 }, 87 { "mod", 300000000, 0 }, 88 { "ram", 0, 400000000 }, 89 }, 90 .esr = ESR_H6, 91 .prng = CE_ALG_PRNG_V2, 92 .trng = CE_ALG_TRNG_V2, 93}; 94 95static const struct ce_variant ce_a64_variant = { 96 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 97 }, 98 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 99 CE_ID_NOTSUPP, CE_ID_NOTSUPP 100 }, 101 .op_mode = { CE_OP_ECB, CE_OP_CBC 102 }, 103 .ce_clks = { 104 { "bus", 0, 200000000 }, 105 { "mod", 300000000, 0 }, 106 }, 107 .esr = ESR_A64, 108 .prng = CE_ALG_PRNG, 109 .trng = CE_ID_NOTSUPP, 110}; 111 112static const struct ce_variant ce_d1_variant = { 113 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 114 }, 115 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 116 CE_ALG_SHA384, CE_ALG_SHA512 117 }, 118 .op_mode = { CE_OP_ECB, CE_OP_CBC 119 }, 120 .ce_clks = { 121 { "bus", 0, 200000000 }, 122 { "mod", 300000000, 0 }, 123 { "ram", 0, 400000000 }, 124 { "trng", 0, 0 }, 125 }, 126 .esr = ESR_D1, 127 .prng = CE_ALG_PRNG, 128 .trng = CE_ALG_TRNG, 129}; 130 131static const struct ce_variant ce_r40_variant = { 132 .alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES, 133 }, 134 .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256, 135 CE_ID_NOTSUPP, CE_ID_NOTSUPP 136 }, 137 .op_mode = { CE_OP_ECB, CE_OP_CBC 138 }, 139 .ce_clks = { 140 { "bus", 0, 200000000 }, 141 { "mod", 300000000, 0 }, 142 }, 143 .esr = ESR_R40, 144 .prng = CE_ALG_PRNG, 145 .trng = CE_ID_NOTSUPP, 146}; 147 148/* 149 * sun8i_ce_get_engine_number() get the next channel slot 150 * This is a simple round-robin way of getting the next channel 151 * The flow 3 is reserve for xRNG operations 152 */ 153int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce) 154{ 155 return atomic_inc_return(&ce->flow) % (MAXFLOW - 1); 156} 157 158int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name) 159{ 160 u32 v; 161 int err = 0; 162 struct ce_task *cet = ce->chanlist[flow].tl; 163 164#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 165 ce->chanlist[flow].stat_req++; 166#endif 167 168 mutex_lock(&ce->mlock); 169 170 v = readl(ce->base + CE_ICR); 171 v |= 1 << flow; 172 writel(v, ce->base + CE_ICR); 173 174 reinit_completion(&ce->chanlist[flow].complete); 175 writel(ce->chanlist[flow].t_phy, ce->base + CE_TDQ); 176 177 ce->chanlist[flow].status = 0; 178 /* Be sure all data is written before enabling the task */ 179 wmb(); 180 181 /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored 182 * on older SoCs, we have no reason to complicate things. 183 */ 184 v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8); 185 writel(v, ce->base + CE_TLR); 186 mutex_unlock(&ce->mlock); 187 188 wait_for_completion_interruptible_timeout(&ce->chanlist[flow].complete, 189 msecs_to_jiffies(ce->chanlist[flow].timeout)); 190 191 if (ce->chanlist[flow].status == 0) { 192 dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name, 193 ce->chanlist[flow].timeout, flow); 194 err = -EFAULT; 195 } 196 /* No need to lock for this read, the channel is locked so 197 * nothing could modify the error value for this channel 198 */ 199 v = readl(ce->base + CE_ESR); 200 switch (ce->variant->esr) { 201 case ESR_H3: 202 /* Sadly, the error bit is not per flow */ 203 if (v) { 204 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); 205 err = -EFAULT; 206 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, 207 cet, sizeof(struct ce_task), false); 208 } 209 if (v & CE_ERR_ALGO_NOTSUP) 210 dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); 211 if (v & CE_ERR_DATALEN) 212 dev_err(ce->dev, "CE ERROR: data length error\n"); 213 if (v & CE_ERR_KEYSRAM) 214 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); 215 break; 216 case ESR_A64: 217 case ESR_D1: 218 case ESR_H5: 219 case ESR_R40: 220 v >>= (flow * 4); 221 v &= 0xF; 222 if (v) { 223 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); 224 err = -EFAULT; 225 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, 226 cet, sizeof(struct ce_task), false); 227 } 228 if (v & CE_ERR_ALGO_NOTSUP) 229 dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); 230 if (v & CE_ERR_DATALEN) 231 dev_err(ce->dev, "CE ERROR: data length error\n"); 232 if (v & CE_ERR_KEYSRAM) 233 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); 234 break; 235 case ESR_H6: 236 v >>= (flow * 8); 237 v &= 0xFF; 238 if (v) { 239 dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow); 240 err = -EFAULT; 241 print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4, 242 cet, sizeof(struct ce_task), false); 243 } 244 if (v & CE_ERR_ALGO_NOTSUP) 245 dev_err(ce->dev, "CE ERROR: algorithm not supported\n"); 246 if (v & CE_ERR_DATALEN) 247 dev_err(ce->dev, "CE ERROR: data length error\n"); 248 if (v & CE_ERR_KEYSRAM) 249 dev_err(ce->dev, "CE ERROR: keysram access error for AES\n"); 250 if (v & CE_ERR_ADDR_INVALID) 251 dev_err(ce->dev, "CE ERROR: address invalid\n"); 252 if (v & CE_ERR_KEYLADDER) 253 dev_err(ce->dev, "CE ERROR: key ladder configuration error\n"); 254 break; 255 } 256 257 return err; 258} 259 260static irqreturn_t ce_irq_handler(int irq, void *data) 261{ 262 struct sun8i_ce_dev *ce = (struct sun8i_ce_dev *)data; 263 int flow = 0; 264 u32 p; 265 266 p = readl(ce->base + CE_ISR); 267 for (flow = 0; flow < MAXFLOW; flow++) { 268 if (p & (BIT(flow))) { 269 writel(BIT(flow), ce->base + CE_ISR); 270 ce->chanlist[flow].status = 1; 271 complete(&ce->chanlist[flow].complete); 272 } 273 } 274 275 return IRQ_HANDLED; 276} 277 278static struct sun8i_ce_alg_template ce_algs[] = { 279{ 280 .type = CRYPTO_ALG_TYPE_SKCIPHER, 281 .ce_algo_id = CE_ID_CIPHER_AES, 282 .ce_blockmode = CE_ID_OP_CBC, 283 .alg.skcipher.base = { 284 .base = { 285 .cra_name = "cbc(aes)", 286 .cra_driver_name = "cbc-aes-sun8i-ce", 287 .cra_priority = 400, 288 .cra_blocksize = AES_BLOCK_SIZE, 289 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 290 CRYPTO_ALG_ASYNC | 291 CRYPTO_ALG_NEED_FALLBACK, 292 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 293 .cra_module = THIS_MODULE, 294 .cra_alignmask = 0xf, 295 .cra_init = sun8i_ce_cipher_init, 296 .cra_exit = sun8i_ce_cipher_exit, 297 }, 298 .min_keysize = AES_MIN_KEY_SIZE, 299 .max_keysize = AES_MAX_KEY_SIZE, 300 .ivsize = AES_BLOCK_SIZE, 301 .setkey = sun8i_ce_aes_setkey, 302 .encrypt = sun8i_ce_skencrypt, 303 .decrypt = sun8i_ce_skdecrypt, 304 }, 305 .alg.skcipher.op = { 306 .do_one_request = sun8i_ce_cipher_do_one, 307 }, 308}, 309{ 310 .type = CRYPTO_ALG_TYPE_SKCIPHER, 311 .ce_algo_id = CE_ID_CIPHER_AES, 312 .ce_blockmode = CE_ID_OP_ECB, 313 .alg.skcipher.base = { 314 .base = { 315 .cra_name = "ecb(aes)", 316 .cra_driver_name = "ecb-aes-sun8i-ce", 317 .cra_priority = 400, 318 .cra_blocksize = AES_BLOCK_SIZE, 319 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 320 CRYPTO_ALG_ASYNC | 321 CRYPTO_ALG_NEED_FALLBACK, 322 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 323 .cra_module = THIS_MODULE, 324 .cra_alignmask = 0xf, 325 .cra_init = sun8i_ce_cipher_init, 326 .cra_exit = sun8i_ce_cipher_exit, 327 }, 328 .min_keysize = AES_MIN_KEY_SIZE, 329 .max_keysize = AES_MAX_KEY_SIZE, 330 .setkey = sun8i_ce_aes_setkey, 331 .encrypt = sun8i_ce_skencrypt, 332 .decrypt = sun8i_ce_skdecrypt, 333 }, 334 .alg.skcipher.op = { 335 .do_one_request = sun8i_ce_cipher_do_one, 336 }, 337}, 338{ 339 .type = CRYPTO_ALG_TYPE_SKCIPHER, 340 .ce_algo_id = CE_ID_CIPHER_DES3, 341 .ce_blockmode = CE_ID_OP_CBC, 342 .alg.skcipher.base = { 343 .base = { 344 .cra_name = "cbc(des3_ede)", 345 .cra_driver_name = "cbc-des3-sun8i-ce", 346 .cra_priority = 400, 347 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 348 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 349 CRYPTO_ALG_ASYNC | 350 CRYPTO_ALG_NEED_FALLBACK, 351 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 352 .cra_module = THIS_MODULE, 353 .cra_alignmask = 0xf, 354 .cra_init = sun8i_ce_cipher_init, 355 .cra_exit = sun8i_ce_cipher_exit, 356 }, 357 .min_keysize = DES3_EDE_KEY_SIZE, 358 .max_keysize = DES3_EDE_KEY_SIZE, 359 .ivsize = DES3_EDE_BLOCK_SIZE, 360 .setkey = sun8i_ce_des3_setkey, 361 .encrypt = sun8i_ce_skencrypt, 362 .decrypt = sun8i_ce_skdecrypt, 363 }, 364 .alg.skcipher.op = { 365 .do_one_request = sun8i_ce_cipher_do_one, 366 }, 367}, 368{ 369 .type = CRYPTO_ALG_TYPE_SKCIPHER, 370 .ce_algo_id = CE_ID_CIPHER_DES3, 371 .ce_blockmode = CE_ID_OP_ECB, 372 .alg.skcipher.base = { 373 .base = { 374 .cra_name = "ecb(des3_ede)", 375 .cra_driver_name = "ecb-des3-sun8i-ce", 376 .cra_priority = 400, 377 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 378 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 379 CRYPTO_ALG_ASYNC | 380 CRYPTO_ALG_NEED_FALLBACK, 381 .cra_ctxsize = sizeof(struct sun8i_cipher_tfm_ctx), 382 .cra_module = THIS_MODULE, 383 .cra_alignmask = 0xf, 384 .cra_init = sun8i_ce_cipher_init, 385 .cra_exit = sun8i_ce_cipher_exit, 386 }, 387 .min_keysize = DES3_EDE_KEY_SIZE, 388 .max_keysize = DES3_EDE_KEY_SIZE, 389 .setkey = sun8i_ce_des3_setkey, 390 .encrypt = sun8i_ce_skencrypt, 391 .decrypt = sun8i_ce_skdecrypt, 392 }, 393 .alg.skcipher.op = { 394 .do_one_request = sun8i_ce_cipher_do_one, 395 }, 396}, 397#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH 398{ .type = CRYPTO_ALG_TYPE_AHASH, 399 .ce_algo_id = CE_ID_HASH_MD5, 400 .alg.hash.base = { 401 .init = sun8i_ce_hash_init, 402 .update = sun8i_ce_hash_update, 403 .final = sun8i_ce_hash_final, 404 .finup = sun8i_ce_hash_finup, 405 .digest = sun8i_ce_hash_digest, 406 .export = sun8i_ce_hash_export, 407 .import = sun8i_ce_hash_import, 408 .init_tfm = sun8i_ce_hash_init_tfm, 409 .exit_tfm = sun8i_ce_hash_exit_tfm, 410 .halg = { 411 .digestsize = MD5_DIGEST_SIZE, 412 .statesize = sizeof(struct md5_state), 413 .base = { 414 .cra_name = "md5", 415 .cra_driver_name = "md5-sun8i-ce", 416 .cra_priority = 300, 417 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 418 CRYPTO_ALG_ASYNC | 419 CRYPTO_ALG_NEED_FALLBACK, 420 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 421 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 422 .cra_module = THIS_MODULE, 423 } 424 } 425 }, 426 .alg.hash.op = { 427 .do_one_request = sun8i_ce_hash_run, 428 }, 429 430}, 431{ .type = CRYPTO_ALG_TYPE_AHASH, 432 .ce_algo_id = CE_ID_HASH_SHA1, 433 .alg.hash.base = { 434 .init = sun8i_ce_hash_init, 435 .update = sun8i_ce_hash_update, 436 .final = sun8i_ce_hash_final, 437 .finup = sun8i_ce_hash_finup, 438 .digest = sun8i_ce_hash_digest, 439 .export = sun8i_ce_hash_export, 440 .import = sun8i_ce_hash_import, 441 .init_tfm = sun8i_ce_hash_init_tfm, 442 .exit_tfm = sun8i_ce_hash_exit_tfm, 443 .halg = { 444 .digestsize = SHA1_DIGEST_SIZE, 445 .statesize = sizeof(struct sha1_state), 446 .base = { 447 .cra_name = "sha1", 448 .cra_driver_name = "sha1-sun8i-ce", 449 .cra_priority = 300, 450 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 451 CRYPTO_ALG_ASYNC | 452 CRYPTO_ALG_NEED_FALLBACK, 453 .cra_blocksize = SHA1_BLOCK_SIZE, 454 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 455 .cra_module = THIS_MODULE, 456 } 457 } 458 }, 459 .alg.hash.op = { 460 .do_one_request = sun8i_ce_hash_run, 461 }, 462}, 463{ .type = CRYPTO_ALG_TYPE_AHASH, 464 .ce_algo_id = CE_ID_HASH_SHA224, 465 .alg.hash.base = { 466 .init = sun8i_ce_hash_init, 467 .update = sun8i_ce_hash_update, 468 .final = sun8i_ce_hash_final, 469 .finup = sun8i_ce_hash_finup, 470 .digest = sun8i_ce_hash_digest, 471 .export = sun8i_ce_hash_export, 472 .import = sun8i_ce_hash_import, 473 .init_tfm = sun8i_ce_hash_init_tfm, 474 .exit_tfm = sun8i_ce_hash_exit_tfm, 475 .halg = { 476 .digestsize = SHA224_DIGEST_SIZE, 477 .statesize = sizeof(struct sha256_state), 478 .base = { 479 .cra_name = "sha224", 480 .cra_driver_name = "sha224-sun8i-ce", 481 .cra_priority = 300, 482 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 483 CRYPTO_ALG_ASYNC | 484 CRYPTO_ALG_NEED_FALLBACK, 485 .cra_blocksize = SHA224_BLOCK_SIZE, 486 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 487 .cra_module = THIS_MODULE, 488 } 489 } 490 }, 491 .alg.hash.op = { 492 .do_one_request = sun8i_ce_hash_run, 493 }, 494}, 495{ .type = CRYPTO_ALG_TYPE_AHASH, 496 .ce_algo_id = CE_ID_HASH_SHA256, 497 .alg.hash.base = { 498 .init = sun8i_ce_hash_init, 499 .update = sun8i_ce_hash_update, 500 .final = sun8i_ce_hash_final, 501 .finup = sun8i_ce_hash_finup, 502 .digest = sun8i_ce_hash_digest, 503 .export = sun8i_ce_hash_export, 504 .import = sun8i_ce_hash_import, 505 .init_tfm = sun8i_ce_hash_init_tfm, 506 .exit_tfm = sun8i_ce_hash_exit_tfm, 507 .halg = { 508 .digestsize = SHA256_DIGEST_SIZE, 509 .statesize = sizeof(struct sha256_state), 510 .base = { 511 .cra_name = "sha256", 512 .cra_driver_name = "sha256-sun8i-ce", 513 .cra_priority = 300, 514 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 515 CRYPTO_ALG_ASYNC | 516 CRYPTO_ALG_NEED_FALLBACK, 517 .cra_blocksize = SHA256_BLOCK_SIZE, 518 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 519 .cra_module = THIS_MODULE, 520 } 521 } 522 }, 523 .alg.hash.op = { 524 .do_one_request = sun8i_ce_hash_run, 525 }, 526}, 527{ .type = CRYPTO_ALG_TYPE_AHASH, 528 .ce_algo_id = CE_ID_HASH_SHA384, 529 .alg.hash.base = { 530 .init = sun8i_ce_hash_init, 531 .update = sun8i_ce_hash_update, 532 .final = sun8i_ce_hash_final, 533 .finup = sun8i_ce_hash_finup, 534 .digest = sun8i_ce_hash_digest, 535 .export = sun8i_ce_hash_export, 536 .import = sun8i_ce_hash_import, 537 .init_tfm = sun8i_ce_hash_init_tfm, 538 .exit_tfm = sun8i_ce_hash_exit_tfm, 539 .halg = { 540 .digestsize = SHA384_DIGEST_SIZE, 541 .statesize = sizeof(struct sha512_state), 542 .base = { 543 .cra_name = "sha384", 544 .cra_driver_name = "sha384-sun8i-ce", 545 .cra_priority = 300, 546 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 547 CRYPTO_ALG_ASYNC | 548 CRYPTO_ALG_NEED_FALLBACK, 549 .cra_blocksize = SHA384_BLOCK_SIZE, 550 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 551 .cra_module = THIS_MODULE, 552 } 553 } 554 }, 555 .alg.hash.op = { 556 .do_one_request = sun8i_ce_hash_run, 557 }, 558}, 559{ .type = CRYPTO_ALG_TYPE_AHASH, 560 .ce_algo_id = CE_ID_HASH_SHA512, 561 .alg.hash.base = { 562 .init = sun8i_ce_hash_init, 563 .update = sun8i_ce_hash_update, 564 .final = sun8i_ce_hash_final, 565 .finup = sun8i_ce_hash_finup, 566 .digest = sun8i_ce_hash_digest, 567 .export = sun8i_ce_hash_export, 568 .import = sun8i_ce_hash_import, 569 .init_tfm = sun8i_ce_hash_init_tfm, 570 .exit_tfm = sun8i_ce_hash_exit_tfm, 571 .halg = { 572 .digestsize = SHA512_DIGEST_SIZE, 573 .statesize = sizeof(struct sha512_state), 574 .base = { 575 .cra_name = "sha512", 576 .cra_driver_name = "sha512-sun8i-ce", 577 .cra_priority = 300, 578 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 579 CRYPTO_ALG_ASYNC | 580 CRYPTO_ALG_NEED_FALLBACK, 581 .cra_blocksize = SHA512_BLOCK_SIZE, 582 .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx), 583 .cra_module = THIS_MODULE, 584 } 585 } 586 }, 587 .alg.hash.op = { 588 .do_one_request = sun8i_ce_hash_run, 589 }, 590}, 591#endif 592#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG 593{ 594 .type = CRYPTO_ALG_TYPE_RNG, 595 .alg.rng = { 596 .base = { 597 .cra_name = "stdrng", 598 .cra_driver_name = "sun8i-ce-prng", 599 .cra_priority = 300, 600 .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx), 601 .cra_module = THIS_MODULE, 602 .cra_init = sun8i_ce_prng_init, 603 .cra_exit = sun8i_ce_prng_exit, 604 }, 605 .generate = sun8i_ce_prng_generate, 606 .seed = sun8i_ce_prng_seed, 607 .seedsize = PRNG_SEED_SIZE, 608 } 609}, 610#endif 611}; 612 613static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) 614{ 615 struct sun8i_ce_dev *ce __maybe_unused = seq->private; 616 unsigned int i; 617 618 for (i = 0; i < MAXFLOW; i++) 619 seq_printf(seq, "Channel %d: nreq %lu\n", i, 620#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 621 ce->chanlist[i].stat_req); 622#else 623 0ul); 624#endif 625 626 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 627 if (!ce_algs[i].ce) 628 continue; 629 switch (ce_algs[i].type) { 630 case CRYPTO_ALG_TYPE_SKCIPHER: 631 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", 632 ce_algs[i].alg.skcipher.base.base.cra_driver_name, 633 ce_algs[i].alg.skcipher.base.base.cra_name, 634 ce_algs[i].stat_req, ce_algs[i].stat_fb); 635 seq_printf(seq, "\tLast fallback is: %s\n", 636 ce_algs[i].fbname); 637 seq_printf(seq, "\tFallback due to 0 length: %lu\n", 638 ce_algs[i].stat_fb_len0); 639 seq_printf(seq, "\tFallback due to length !mod16: %lu\n", 640 ce_algs[i].stat_fb_mod16); 641 seq_printf(seq, "\tFallback due to length < IV: %lu\n", 642 ce_algs[i].stat_fb_leniv); 643 seq_printf(seq, "\tFallback due to source alignment: %lu\n", 644 ce_algs[i].stat_fb_srcali); 645 seq_printf(seq, "\tFallback due to dest alignment: %lu\n", 646 ce_algs[i].stat_fb_dstali); 647 seq_printf(seq, "\tFallback due to source length: %lu\n", 648 ce_algs[i].stat_fb_srclen); 649 seq_printf(seq, "\tFallback due to dest length: %lu\n", 650 ce_algs[i].stat_fb_dstlen); 651 seq_printf(seq, "\tFallback due to SG numbers: %lu\n", 652 ce_algs[i].stat_fb_maxsg); 653 break; 654 case CRYPTO_ALG_TYPE_AHASH: 655 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", 656 ce_algs[i].alg.hash.base.halg.base.cra_driver_name, 657 ce_algs[i].alg.hash.base.halg.base.cra_name, 658 ce_algs[i].stat_req, ce_algs[i].stat_fb); 659 seq_printf(seq, "\tLast fallback is: %s\n", 660 ce_algs[i].fbname); 661 seq_printf(seq, "\tFallback due to 0 length: %lu\n", 662 ce_algs[i].stat_fb_len0); 663 seq_printf(seq, "\tFallback due to length: %lu\n", 664 ce_algs[i].stat_fb_srclen); 665 seq_printf(seq, "\tFallback due to alignment: %lu\n", 666 ce_algs[i].stat_fb_srcali); 667 seq_printf(seq, "\tFallback due to SG numbers: %lu\n", 668 ce_algs[i].stat_fb_maxsg); 669 break; 670 case CRYPTO_ALG_TYPE_RNG: 671 seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n", 672 ce_algs[i].alg.rng.base.cra_driver_name, 673 ce_algs[i].alg.rng.base.cra_name, 674 ce_algs[i].stat_req, ce_algs[i].stat_bytes); 675 break; 676 } 677 } 678#if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \ 679 defined(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG) 680 seq_printf(seq, "HWRNG %lu %lu\n", 681 ce->hwrng_stat_req, ce->hwrng_stat_bytes); 682#endif 683 return 0; 684} 685 686DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs); 687 688static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i) 689{ 690 while (i >= 0) { 691 crypto_engine_exit(ce->chanlist[i].engine); 692 if (ce->chanlist[i].tl) 693 dma_free_coherent(ce->dev, sizeof(struct ce_task), 694 ce->chanlist[i].tl, 695 ce->chanlist[i].t_phy); 696 i--; 697 } 698} 699 700/* 701 * Allocate the channel list structure 702 */ 703static int sun8i_ce_allocate_chanlist(struct sun8i_ce_dev *ce) 704{ 705 int i, err; 706 707 ce->chanlist = devm_kcalloc(ce->dev, MAXFLOW, 708 sizeof(struct sun8i_ce_flow), GFP_KERNEL); 709 if (!ce->chanlist) 710 return -ENOMEM; 711 712 for (i = 0; i < MAXFLOW; i++) { 713 init_completion(&ce->chanlist[i].complete); 714 715 ce->chanlist[i].engine = crypto_engine_alloc_init(ce->dev, true); 716 if (!ce->chanlist[i].engine) { 717 dev_err(ce->dev, "Cannot allocate engine\n"); 718 i--; 719 err = -ENOMEM; 720 goto error_engine; 721 } 722 err = crypto_engine_start(ce->chanlist[i].engine); 723 if (err) { 724 dev_err(ce->dev, "Cannot start engine\n"); 725 goto error_engine; 726 } 727 ce->chanlist[i].tl = dma_alloc_coherent(ce->dev, 728 sizeof(struct ce_task), 729 &ce->chanlist[i].t_phy, 730 GFP_KERNEL); 731 if (!ce->chanlist[i].tl) { 732 dev_err(ce->dev, "Cannot get DMA memory for task %d\n", 733 i); 734 err = -ENOMEM; 735 goto error_engine; 736 } 737 ce->chanlist[i].bounce_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, 738 GFP_KERNEL | GFP_DMA); 739 if (!ce->chanlist[i].bounce_iv) { 740 err = -ENOMEM; 741 goto error_engine; 742 } 743 ce->chanlist[i].backup_iv = devm_kmalloc(ce->dev, AES_BLOCK_SIZE, 744 GFP_KERNEL); 745 if (!ce->chanlist[i].backup_iv) { 746 err = -ENOMEM; 747 goto error_engine; 748 } 749 } 750 return 0; 751error_engine: 752 sun8i_ce_free_chanlist(ce, i); 753 return err; 754} 755 756/* 757 * Power management strategy: The device is suspended unless a TFM exists for 758 * one of the algorithms proposed by this driver. 759 */ 760static int sun8i_ce_pm_suspend(struct device *dev) 761{ 762 struct sun8i_ce_dev *ce = dev_get_drvdata(dev); 763 int i; 764 765 reset_control_assert(ce->reset); 766 for (i = 0; i < CE_MAX_CLOCKS; i++) 767 clk_disable_unprepare(ce->ceclks[i]); 768 return 0; 769} 770 771static int sun8i_ce_pm_resume(struct device *dev) 772{ 773 struct sun8i_ce_dev *ce = dev_get_drvdata(dev); 774 int err, i; 775 776 for (i = 0; i < CE_MAX_CLOCKS; i++) { 777 if (!ce->variant->ce_clks[i].name) 778 continue; 779 err = clk_prepare_enable(ce->ceclks[i]); 780 if (err) { 781 dev_err(ce->dev, "Cannot prepare_enable %s\n", 782 ce->variant->ce_clks[i].name); 783 goto error; 784 } 785 } 786 err = reset_control_deassert(ce->reset); 787 if (err) { 788 dev_err(ce->dev, "Cannot deassert reset control\n"); 789 goto error; 790 } 791 return 0; 792error: 793 sun8i_ce_pm_suspend(dev); 794 return err; 795} 796 797static const struct dev_pm_ops sun8i_ce_pm_ops = { 798 SET_RUNTIME_PM_OPS(sun8i_ce_pm_suspend, sun8i_ce_pm_resume, NULL) 799}; 800 801static int sun8i_ce_pm_init(struct sun8i_ce_dev *ce) 802{ 803 int err; 804 805 pm_runtime_use_autosuspend(ce->dev); 806 pm_runtime_set_autosuspend_delay(ce->dev, 2000); 807 808 err = pm_runtime_set_suspended(ce->dev); 809 if (err) 810 return err; 811 pm_runtime_enable(ce->dev); 812 return err; 813} 814 815static void sun8i_ce_pm_exit(struct sun8i_ce_dev *ce) 816{ 817 pm_runtime_disable(ce->dev); 818} 819 820static int sun8i_ce_get_clks(struct sun8i_ce_dev *ce) 821{ 822 unsigned long cr; 823 int err, i; 824 825 for (i = 0; i < CE_MAX_CLOCKS; i++) { 826 if (!ce->variant->ce_clks[i].name) 827 continue; 828 ce->ceclks[i] = devm_clk_get(ce->dev, ce->variant->ce_clks[i].name); 829 if (IS_ERR(ce->ceclks[i])) { 830 err = PTR_ERR(ce->ceclks[i]); 831 dev_err(ce->dev, "Cannot get %s CE clock err=%d\n", 832 ce->variant->ce_clks[i].name, err); 833 return err; 834 } 835 cr = clk_get_rate(ce->ceclks[i]); 836 if (!cr) 837 return -EINVAL; 838 if (ce->variant->ce_clks[i].freq > 0 && 839 cr != ce->variant->ce_clks[i].freq) { 840 dev_info(ce->dev, "Set %s clock to %lu (%lu Mhz) from %lu (%lu Mhz)\n", 841 ce->variant->ce_clks[i].name, 842 ce->variant->ce_clks[i].freq, 843 ce->variant->ce_clks[i].freq / 1000000, 844 cr, cr / 1000000); 845 err = clk_set_rate(ce->ceclks[i], ce->variant->ce_clks[i].freq); 846 if (err) 847 dev_err(ce->dev, "Fail to set %s clk speed to %lu hz\n", 848 ce->variant->ce_clks[i].name, 849 ce->variant->ce_clks[i].freq); 850 } 851 if (ce->variant->ce_clks[i].max_freq > 0 && 852 cr > ce->variant->ce_clks[i].max_freq) 853 dev_warn(ce->dev, "Frequency for %s (%lu hz) is higher than datasheet's recommendation (%lu hz)", 854 ce->variant->ce_clks[i].name, cr, 855 ce->variant->ce_clks[i].max_freq); 856 } 857 return 0; 858} 859 860static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) 861{ 862 int ce_method, err, id; 863 unsigned int i; 864 865 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 866 ce_algs[i].ce = ce; 867 switch (ce_algs[i].type) { 868 case CRYPTO_ALG_TYPE_SKCIPHER: 869 id = ce_algs[i].ce_algo_id; 870 ce_method = ce->variant->alg_cipher[id]; 871 if (ce_method == CE_ID_NOTSUPP) { 872 dev_dbg(ce->dev, 873 "DEBUG: Algo of %s not supported\n", 874 ce_algs[i].alg.skcipher.base.base.cra_name); 875 ce_algs[i].ce = NULL; 876 break; 877 } 878 id = ce_algs[i].ce_blockmode; 879 ce_method = ce->variant->op_mode[id]; 880 if (ce_method == CE_ID_NOTSUPP) { 881 dev_dbg(ce->dev, "DEBUG: Blockmode of %s not supported\n", 882 ce_algs[i].alg.skcipher.base.base.cra_name); 883 ce_algs[i].ce = NULL; 884 break; 885 } 886 dev_info(ce->dev, "Register %s\n", 887 ce_algs[i].alg.skcipher.base.base.cra_name); 888 err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher); 889 if (err) { 890 dev_err(ce->dev, "ERROR: Fail to register %s\n", 891 ce_algs[i].alg.skcipher.base.base.cra_name); 892 ce_algs[i].ce = NULL; 893 return err; 894 } 895 break; 896 case CRYPTO_ALG_TYPE_AHASH: 897 id = ce_algs[i].ce_algo_id; 898 ce_method = ce->variant->alg_hash[id]; 899 if (ce_method == CE_ID_NOTSUPP) { 900 dev_info(ce->dev, 901 "DEBUG: Algo of %s not supported\n", 902 ce_algs[i].alg.hash.base.halg.base.cra_name); 903 ce_algs[i].ce = NULL; 904 break; 905 } 906 dev_info(ce->dev, "Register %s\n", 907 ce_algs[i].alg.hash.base.halg.base.cra_name); 908 err = crypto_engine_register_ahash(&ce_algs[i].alg.hash); 909 if (err) { 910 dev_err(ce->dev, "ERROR: Fail to register %s\n", 911 ce_algs[i].alg.hash.base.halg.base.cra_name); 912 ce_algs[i].ce = NULL; 913 return err; 914 } 915 break; 916 case CRYPTO_ALG_TYPE_RNG: 917 if (ce->variant->prng == CE_ID_NOTSUPP) { 918 dev_info(ce->dev, 919 "DEBUG: Algo of %s not supported\n", 920 ce_algs[i].alg.rng.base.cra_name); 921 ce_algs[i].ce = NULL; 922 break; 923 } 924 dev_info(ce->dev, "Register %s\n", 925 ce_algs[i].alg.rng.base.cra_name); 926 err = crypto_register_rng(&ce_algs[i].alg.rng); 927 if (err) { 928 dev_err(ce->dev, "Fail to register %s\n", 929 ce_algs[i].alg.rng.base.cra_name); 930 ce_algs[i].ce = NULL; 931 } 932 break; 933 default: 934 ce_algs[i].ce = NULL; 935 dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); 936 } 937 } 938 return 0; 939} 940 941static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) 942{ 943 unsigned int i; 944 945 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 946 if (!ce_algs[i].ce) 947 continue; 948 switch (ce_algs[i].type) { 949 case CRYPTO_ALG_TYPE_SKCIPHER: 950 dev_info(ce->dev, "Unregister %d %s\n", i, 951 ce_algs[i].alg.skcipher.base.base.cra_name); 952 crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher); 953 break; 954 case CRYPTO_ALG_TYPE_AHASH: 955 dev_info(ce->dev, "Unregister %d %s\n", i, 956 ce_algs[i].alg.hash.base.halg.base.cra_name); 957 crypto_engine_unregister_ahash(&ce_algs[i].alg.hash); 958 break; 959 case CRYPTO_ALG_TYPE_RNG: 960 dev_info(ce->dev, "Unregister %d %s\n", i, 961 ce_algs[i].alg.rng.base.cra_name); 962 crypto_unregister_rng(&ce_algs[i].alg.rng); 963 break; 964 } 965 } 966} 967 968static int sun8i_ce_probe(struct platform_device *pdev) 969{ 970 struct sun8i_ce_dev *ce; 971 int err, irq; 972 u32 v; 973 974 ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); 975 if (!ce) 976 return -ENOMEM; 977 978 ce->dev = &pdev->dev; 979 platform_set_drvdata(pdev, ce); 980 981 ce->variant = of_device_get_match_data(&pdev->dev); 982 if (!ce->variant) { 983 dev_err(&pdev->dev, "Missing Crypto Engine variant\n"); 984 return -EINVAL; 985 } 986 987 ce->base = devm_platform_ioremap_resource(pdev, 0); 988 if (IS_ERR(ce->base)) 989 return PTR_ERR(ce->base); 990 991 err = sun8i_ce_get_clks(ce); 992 if (err) 993 return err; 994 995 /* Get Non Secure IRQ */ 996 irq = platform_get_irq(pdev, 0); 997 if (irq < 0) 998 return irq; 999 1000 ce->reset = devm_reset_control_get(&pdev->dev, NULL); 1001 if (IS_ERR(ce->reset)) 1002 return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), 1003 "No reset control found\n"); 1004 1005 mutex_init(&ce->mlock); 1006 mutex_init(&ce->rnglock); 1007 1008 err = sun8i_ce_allocate_chanlist(ce); 1009 if (err) 1010 return err; 1011 1012 err = sun8i_ce_pm_init(ce); 1013 if (err) 1014 goto error_pm; 1015 1016 err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, 1017 "sun8i-ce-ns", ce); 1018 if (err) { 1019 dev_err(ce->dev, "Cannot request CryptoEngine Non-secure IRQ (err=%d)\n", err); 1020 goto error_irq; 1021 } 1022 1023 err = sun8i_ce_register_algs(ce); 1024 if (err) 1025 goto error_alg; 1026 1027 err = pm_runtime_resume_and_get(ce->dev); 1028 if (err < 0) 1029 goto error_alg; 1030 1031#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG 1032 sun8i_ce_hwrng_register(ce); 1033#endif 1034 1035 v = readl(ce->base + CE_CTR); 1036 v >>= CE_DIE_ID_SHIFT; 1037 v &= CE_DIE_ID_MASK; 1038 dev_info(&pdev->dev, "CryptoEngine Die ID %x\n", v); 1039 1040 pm_runtime_put_sync(ce->dev); 1041 1042 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) { 1043 struct dentry *dbgfs_dir __maybe_unused; 1044 struct dentry *dbgfs_stats __maybe_unused; 1045 1046 /* Ignore error of debugfs */ 1047 dbgfs_dir = debugfs_create_dir("sun8i-ce", NULL); 1048 dbgfs_stats = debugfs_create_file("stats", 0444, 1049 dbgfs_dir, ce, 1050 &sun8i_ce_debugfs_fops); 1051 1052#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 1053 ce->dbgfs_dir = dbgfs_dir; 1054 ce->dbgfs_stats = dbgfs_stats; 1055#endif 1056 } 1057 1058 return 0; 1059error_alg: 1060 sun8i_ce_unregister_algs(ce); 1061error_irq: 1062 sun8i_ce_pm_exit(ce); 1063error_pm: 1064 sun8i_ce_free_chanlist(ce, MAXFLOW - 1); 1065 return err; 1066} 1067 1068static void sun8i_ce_remove(struct platform_device *pdev) 1069{ 1070 struct sun8i_ce_dev *ce = platform_get_drvdata(pdev); 1071 1072#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG 1073 sun8i_ce_hwrng_unregister(ce); 1074#endif 1075 1076 sun8i_ce_unregister_algs(ce); 1077 1078#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 1079 debugfs_remove_recursive(ce->dbgfs_dir); 1080#endif 1081 1082 sun8i_ce_free_chanlist(ce, MAXFLOW - 1); 1083 1084 sun8i_ce_pm_exit(ce); 1085} 1086 1087static const struct of_device_id sun8i_ce_crypto_of_match_table[] = { 1088 { .compatible = "allwinner,sun8i-h3-crypto", 1089 .data = &ce_h3_variant }, 1090 { .compatible = "allwinner,sun8i-r40-crypto", 1091 .data = &ce_r40_variant }, 1092 { .compatible = "allwinner,sun20i-d1-crypto", 1093 .data = &ce_d1_variant }, 1094 { .compatible = "allwinner,sun50i-a64-crypto", 1095 .data = &ce_a64_variant }, 1096 { .compatible = "allwinner,sun50i-h5-crypto", 1097 .data = &ce_h5_variant }, 1098 { .compatible = "allwinner,sun50i-h6-crypto", 1099 .data = &ce_h6_variant }, 1100 {} 1101}; 1102MODULE_DEVICE_TABLE(of, sun8i_ce_crypto_of_match_table); 1103 1104static struct platform_driver sun8i_ce_driver = { 1105 .probe = sun8i_ce_probe, 1106 .remove_new = sun8i_ce_remove, 1107 .driver = { 1108 .name = "sun8i-ce", 1109 .pm = &sun8i_ce_pm_ops, 1110 .of_match_table = sun8i_ce_crypto_of_match_table, 1111 }, 1112}; 1113 1114module_platform_driver(sun8i_ce_driver); 1115 1116MODULE_DESCRIPTION("Allwinner Crypto Engine cryptographic offloader"); 1117MODULE_LICENSE("GPL"); 1118MODULE_AUTHOR("Corentin Labbe <clabbe.montjoie@gmail.com>"); 1119