1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Software async crypto daemon. 4 * 5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 6 * 7 * Added AEAD support to cryptd. 8 * Authors: Tadeusz Struk (tadeusz.struk@intel.com) 9 * Adrian Hoban <adrian.hoban@intel.com> 10 * Gabriele Paoloni <gabriele.paoloni@intel.com> 11 * Aidan O'Mahony (aidan.o.mahony@intel.com) 12 * Copyright (c) 2010, Intel Corporation. 13 */ 14 15#include <crypto/internal/hash.h> 16#include <crypto/internal/aead.h> 17#include <crypto/internal/skcipher.h> 18#include <crypto/cryptd.h> 19#include <linux/refcount.h> 20#include <linux/err.h> 21#include <linux/init.h> 22#include <linux/kernel.h> 23#include <linux/list.h> 24#include <linux/module.h> 25#include <linux/scatterlist.h> 26#include <linux/sched.h> 27#include <linux/slab.h> 28#include <linux/workqueue.h> 29 30static unsigned int cryptd_max_cpu_qlen = 1000; 31module_param(cryptd_max_cpu_qlen, uint, 0); 32MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); 33 34static struct workqueue_struct *cryptd_wq; 35 36struct cryptd_cpu_queue { 37 struct crypto_queue queue; 38 struct work_struct work; 39}; 40 41struct cryptd_queue { 42 /* 43 * Protected by disabling BH to allow enqueueing from softinterrupt and 44 * dequeuing from kworker (cryptd_queue_worker()). 45 */ 46 struct cryptd_cpu_queue __percpu *cpu_queue; 47}; 48 49struct cryptd_instance_ctx { 50 struct crypto_spawn spawn; 51 struct cryptd_queue *queue; 52}; 53 54struct skcipherd_instance_ctx { 55 struct crypto_skcipher_spawn spawn; 56 struct cryptd_queue *queue; 57}; 58 59struct hashd_instance_ctx { 60 struct crypto_shash_spawn spawn; 61 struct cryptd_queue *queue; 62}; 63 64struct aead_instance_ctx { 65 struct crypto_aead_spawn aead_spawn; 66 struct cryptd_queue *queue; 67}; 68 69struct cryptd_skcipher_ctx { 70 refcount_t refcnt; 71 struct crypto_skcipher *child; 72}; 73 74struct cryptd_skcipher_request_ctx { 75 struct skcipher_request req; 76}; 77 78struct cryptd_hash_ctx { 79 refcount_t refcnt; 80 struct crypto_shash *child; 81}; 82 83struct cryptd_hash_request_ctx { 84 crypto_completion_t complete; 85 void *data; 86 struct shash_desc desc; 87}; 88 89struct cryptd_aead_ctx { 90 refcount_t refcnt; 91 struct crypto_aead *child; 92}; 93 94struct cryptd_aead_request_ctx { 95 struct aead_request req; 96}; 97 98static void cryptd_queue_worker(struct work_struct *work); 99 100static int cryptd_init_queue(struct cryptd_queue *queue, 101 unsigned int max_cpu_qlen) 102{ 103 int cpu; 104 struct cryptd_cpu_queue *cpu_queue; 105 106 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); 107 if (!queue->cpu_queue) 108 return -ENOMEM; 109 for_each_possible_cpu(cpu) { 110 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 111 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); 112 INIT_WORK(&cpu_queue->work, cryptd_queue_worker); 113 } 114 pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); 115 return 0; 116} 117 118static void cryptd_fini_queue(struct cryptd_queue *queue) 119{ 120 int cpu; 121 struct cryptd_cpu_queue *cpu_queue; 122 123 for_each_possible_cpu(cpu) { 124 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 125 BUG_ON(cpu_queue->queue.qlen); 126 } 127 free_percpu(queue->cpu_queue); 128} 129 130static int cryptd_enqueue_request(struct cryptd_queue *queue, 131 struct crypto_async_request *request) 132{ 133 int err; 134 struct cryptd_cpu_queue *cpu_queue; 135 refcount_t *refcnt; 136 137 local_bh_disable(); 138 cpu_queue = this_cpu_ptr(queue->cpu_queue); 139 err = crypto_enqueue_request(&cpu_queue->queue, request); 140 141 refcnt = crypto_tfm_ctx(request->tfm); 142 143 if (err == -ENOSPC) 144 goto out; 145 146 queue_work_on(smp_processor_id(), cryptd_wq, &cpu_queue->work); 147 148 if (!refcount_read(refcnt)) 149 goto out; 150 151 refcount_inc(refcnt); 152 153out: 154 local_bh_enable(); 155 156 return err; 157} 158 159/* Called in workqueue context, do one real cryption work (via 160 * req->complete) and reschedule itself if there are more work to 161 * do. */ 162static void cryptd_queue_worker(struct work_struct *work) 163{ 164 struct cryptd_cpu_queue *cpu_queue; 165 struct crypto_async_request *req, *backlog; 166 167 cpu_queue = container_of(work, struct cryptd_cpu_queue, work); 168 /* 169 * Only handle one request at a time to avoid hogging crypto workqueue. 170 */ 171 local_bh_disable(); 172 backlog = crypto_get_backlog(&cpu_queue->queue); 173 req = crypto_dequeue_request(&cpu_queue->queue); 174 local_bh_enable(); 175 176 if (!req) 177 return; 178 179 if (backlog) 180 crypto_request_complete(backlog, -EINPROGRESS); 181 crypto_request_complete(req, 0); 182 183 if (cpu_queue->queue.qlen) 184 queue_work(cryptd_wq, &cpu_queue->work); 185} 186 187static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) 188{ 189 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 190 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); 191 return ictx->queue; 192} 193 194static void cryptd_type_and_mask(struct crypto_attr_type *algt, 195 u32 *type, u32 *mask) 196{ 197 /* 198 * cryptd is allowed to wrap internal algorithms, but in that case the 199 * resulting cryptd instance will be marked as internal as well. 200 */ 201 *type = algt->type & CRYPTO_ALG_INTERNAL; 202 *mask = algt->mask & CRYPTO_ALG_INTERNAL; 203 204 /* No point in cryptd wrapping an algorithm that's already async. */ 205 *mask |= CRYPTO_ALG_ASYNC; 206 207 *mask |= crypto_algt_inherited_mask(algt); 208} 209 210static int cryptd_init_instance(struct crypto_instance *inst, 211 struct crypto_alg *alg) 212{ 213 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 214 "cryptd(%s)", 215 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 216 return -ENAMETOOLONG; 217 218 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 219 220 inst->alg.cra_priority = alg->cra_priority + 50; 221 inst->alg.cra_blocksize = alg->cra_blocksize; 222 inst->alg.cra_alignmask = alg->cra_alignmask; 223 224 return 0; 225} 226 227static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, 228 const u8 *key, unsigned int keylen) 229{ 230 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent); 231 struct crypto_skcipher *child = ctx->child; 232 233 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 234 crypto_skcipher_set_flags(child, 235 crypto_skcipher_get_flags(parent) & 236 CRYPTO_TFM_REQ_MASK); 237 return crypto_skcipher_setkey(child, key, keylen); 238} 239 240static struct skcipher_request *cryptd_skcipher_prepare( 241 struct skcipher_request *req, int err) 242{ 243 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); 244 struct skcipher_request *subreq = &rctx->req; 245 struct cryptd_skcipher_ctx *ctx; 246 struct crypto_skcipher *child; 247 248 req->base.complete = subreq->base.complete; 249 req->base.data = subreq->base.data; 250 251 if (unlikely(err == -EINPROGRESS)) 252 return NULL; 253 254 ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 255 child = ctx->child; 256 257 skcipher_request_set_tfm(subreq, child); 258 skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, 259 NULL, NULL); 260 skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 261 req->iv); 262 263 return subreq; 264} 265 266static void cryptd_skcipher_complete(struct skcipher_request *req, int err, 267 crypto_completion_t complete) 268{ 269 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); 270 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 271 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 272 struct skcipher_request *subreq = &rctx->req; 273 int refcnt = refcount_read(&ctx->refcnt); 274 275 local_bh_disable(); 276 skcipher_request_complete(req, err); 277 local_bh_enable(); 278 279 if (unlikely(err == -EINPROGRESS)) { 280 subreq->base.complete = req->base.complete; 281 subreq->base.data = req->base.data; 282 req->base.complete = complete; 283 req->base.data = req; 284 } else if (refcnt && refcount_dec_and_test(&ctx->refcnt)) 285 crypto_free_skcipher(tfm); 286} 287 288static void cryptd_skcipher_encrypt(void *data, int err) 289{ 290 struct skcipher_request *req = data; 291 struct skcipher_request *subreq; 292 293 subreq = cryptd_skcipher_prepare(req, err); 294 if (likely(subreq)) 295 err = crypto_skcipher_encrypt(subreq); 296 297 cryptd_skcipher_complete(req, err, cryptd_skcipher_encrypt); 298} 299 300static void cryptd_skcipher_decrypt(void *data, int err) 301{ 302 struct skcipher_request *req = data; 303 struct skcipher_request *subreq; 304 305 subreq = cryptd_skcipher_prepare(req, err); 306 if (likely(subreq)) 307 err = crypto_skcipher_decrypt(subreq); 308 309 cryptd_skcipher_complete(req, err, cryptd_skcipher_decrypt); 310} 311 312static int cryptd_skcipher_enqueue(struct skcipher_request *req, 313 crypto_completion_t compl) 314{ 315 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); 316 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 317 struct skcipher_request *subreq = &rctx->req; 318 struct cryptd_queue *queue; 319 320 queue = cryptd_get_queue(crypto_skcipher_tfm(tfm)); 321 subreq->base.complete = req->base.complete; 322 subreq->base.data = req->base.data; 323 req->base.complete = compl; 324 req->base.data = req; 325 326 return cryptd_enqueue_request(queue, &req->base); 327} 328 329static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req) 330{ 331 return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt); 332} 333 334static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req) 335{ 336 return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt); 337} 338 339static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm) 340{ 341 struct skcipher_instance *inst = skcipher_alg_instance(tfm); 342 struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst); 343 struct crypto_skcipher_spawn *spawn = &ictx->spawn; 344 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 345 struct crypto_skcipher *cipher; 346 347 cipher = crypto_spawn_skcipher(spawn); 348 if (IS_ERR(cipher)) 349 return PTR_ERR(cipher); 350 351 ctx->child = cipher; 352 crypto_skcipher_set_reqsize( 353 tfm, sizeof(struct cryptd_skcipher_request_ctx) + 354 crypto_skcipher_reqsize(cipher)); 355 return 0; 356} 357 358static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm) 359{ 360 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 361 362 crypto_free_skcipher(ctx->child); 363} 364 365static void cryptd_skcipher_free(struct skcipher_instance *inst) 366{ 367 struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); 368 369 crypto_drop_skcipher(&ctx->spawn); 370 kfree(inst); 371} 372 373static int cryptd_create_skcipher(struct crypto_template *tmpl, 374 struct rtattr **tb, 375 struct crypto_attr_type *algt, 376 struct cryptd_queue *queue) 377{ 378 struct skcipherd_instance_ctx *ctx; 379 struct skcipher_instance *inst; 380 struct skcipher_alg_common *alg; 381 u32 type; 382 u32 mask; 383 int err; 384 385 cryptd_type_and_mask(algt, &type, &mask); 386 387 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 388 if (!inst) 389 return -ENOMEM; 390 391 ctx = skcipher_instance_ctx(inst); 392 ctx->queue = queue; 393 394 err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), 395 crypto_attr_alg_name(tb[1]), type, mask); 396 if (err) 397 goto err_free_inst; 398 399 alg = crypto_spawn_skcipher_alg_common(&ctx->spawn); 400 err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base); 401 if (err) 402 goto err_free_inst; 403 404 inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | 405 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); 406 inst->alg.ivsize = alg->ivsize; 407 inst->alg.chunksize = alg->chunksize; 408 inst->alg.min_keysize = alg->min_keysize; 409 inst->alg.max_keysize = alg->max_keysize; 410 411 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx); 412 413 inst->alg.init = cryptd_skcipher_init_tfm; 414 inst->alg.exit = cryptd_skcipher_exit_tfm; 415 416 inst->alg.setkey = cryptd_skcipher_setkey; 417 inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue; 418 inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue; 419 420 inst->free = cryptd_skcipher_free; 421 422 err = skcipher_register_instance(tmpl, inst); 423 if (err) { 424err_free_inst: 425 cryptd_skcipher_free(inst); 426 } 427 return err; 428} 429 430static int cryptd_hash_init_tfm(struct crypto_ahash *tfm) 431{ 432 struct ahash_instance *inst = ahash_alg_instance(tfm); 433 struct hashd_instance_ctx *ictx = ahash_instance_ctx(inst); 434 struct crypto_shash_spawn *spawn = &ictx->spawn; 435 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 436 struct crypto_shash *hash; 437 438 hash = crypto_spawn_shash(spawn); 439 if (IS_ERR(hash)) 440 return PTR_ERR(hash); 441 442 ctx->child = hash; 443 crypto_ahash_set_reqsize(tfm, 444 sizeof(struct cryptd_hash_request_ctx) + 445 crypto_shash_descsize(hash)); 446 return 0; 447} 448 449static int cryptd_hash_clone_tfm(struct crypto_ahash *ntfm, 450 struct crypto_ahash *tfm) 451{ 452 struct cryptd_hash_ctx *nctx = crypto_ahash_ctx(ntfm); 453 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 454 struct crypto_shash *hash; 455 456 hash = crypto_clone_shash(ctx->child); 457 if (IS_ERR(hash)) 458 return PTR_ERR(hash); 459 460 nctx->child = hash; 461 return 0; 462} 463 464static void cryptd_hash_exit_tfm(struct crypto_ahash *tfm) 465{ 466 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 467 468 crypto_free_shash(ctx->child); 469} 470 471static int cryptd_hash_setkey(struct crypto_ahash *parent, 472 const u8 *key, unsigned int keylen) 473{ 474 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); 475 struct crypto_shash *child = ctx->child; 476 477 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 478 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & 479 CRYPTO_TFM_REQ_MASK); 480 return crypto_shash_setkey(child, key, keylen); 481} 482 483static int cryptd_hash_enqueue(struct ahash_request *req, 484 crypto_completion_t compl) 485{ 486 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 487 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 488 struct cryptd_queue *queue = 489 cryptd_get_queue(crypto_ahash_tfm(tfm)); 490 491 rctx->complete = req->base.complete; 492 rctx->data = req->base.data; 493 req->base.complete = compl; 494 req->base.data = req; 495 496 return cryptd_enqueue_request(queue, &req->base); 497} 498 499static struct shash_desc *cryptd_hash_prepare(struct ahash_request *req, 500 int err) 501{ 502 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 503 504 req->base.complete = rctx->complete; 505 req->base.data = rctx->data; 506 507 if (unlikely(err == -EINPROGRESS)) 508 return NULL; 509 510 return &rctx->desc; 511} 512 513static void cryptd_hash_complete(struct ahash_request *req, int err, 514 crypto_completion_t complete) 515{ 516 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 517 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 518 int refcnt = refcount_read(&ctx->refcnt); 519 520 local_bh_disable(); 521 ahash_request_complete(req, err); 522 local_bh_enable(); 523 524 if (err == -EINPROGRESS) { 525 req->base.complete = complete; 526 req->base.data = req; 527 } else if (refcnt && refcount_dec_and_test(&ctx->refcnt)) 528 crypto_free_ahash(tfm); 529} 530 531static void cryptd_hash_init(void *data, int err) 532{ 533 struct ahash_request *req = data; 534 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 535 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 536 struct crypto_shash *child = ctx->child; 537 struct shash_desc *desc; 538 539 desc = cryptd_hash_prepare(req, err); 540 if (unlikely(!desc)) 541 goto out; 542 543 desc->tfm = child; 544 545 err = crypto_shash_init(desc); 546 547out: 548 cryptd_hash_complete(req, err, cryptd_hash_init); 549} 550 551static int cryptd_hash_init_enqueue(struct ahash_request *req) 552{ 553 return cryptd_hash_enqueue(req, cryptd_hash_init); 554} 555 556static void cryptd_hash_update(void *data, int err) 557{ 558 struct ahash_request *req = data; 559 struct shash_desc *desc; 560 561 desc = cryptd_hash_prepare(req, err); 562 if (likely(desc)) 563 err = shash_ahash_update(req, desc); 564 565 cryptd_hash_complete(req, err, cryptd_hash_update); 566} 567 568static int cryptd_hash_update_enqueue(struct ahash_request *req) 569{ 570 return cryptd_hash_enqueue(req, cryptd_hash_update); 571} 572 573static void cryptd_hash_final(void *data, int err) 574{ 575 struct ahash_request *req = data; 576 struct shash_desc *desc; 577 578 desc = cryptd_hash_prepare(req, err); 579 if (likely(desc)) 580 err = crypto_shash_final(desc, req->result); 581 582 cryptd_hash_complete(req, err, cryptd_hash_final); 583} 584 585static int cryptd_hash_final_enqueue(struct ahash_request *req) 586{ 587 return cryptd_hash_enqueue(req, cryptd_hash_final); 588} 589 590static void cryptd_hash_finup(void *data, int err) 591{ 592 struct ahash_request *req = data; 593 struct shash_desc *desc; 594 595 desc = cryptd_hash_prepare(req, err); 596 if (likely(desc)) 597 err = shash_ahash_finup(req, desc); 598 599 cryptd_hash_complete(req, err, cryptd_hash_finup); 600} 601 602static int cryptd_hash_finup_enqueue(struct ahash_request *req) 603{ 604 return cryptd_hash_enqueue(req, cryptd_hash_finup); 605} 606 607static void cryptd_hash_digest(void *data, int err) 608{ 609 struct ahash_request *req = data; 610 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 611 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 612 struct crypto_shash *child = ctx->child; 613 struct shash_desc *desc; 614 615 desc = cryptd_hash_prepare(req, err); 616 if (unlikely(!desc)) 617 goto out; 618 619 desc->tfm = child; 620 621 err = shash_ahash_digest(req, desc); 622 623out: 624 cryptd_hash_complete(req, err, cryptd_hash_digest); 625} 626 627static int cryptd_hash_digest_enqueue(struct ahash_request *req) 628{ 629 return cryptd_hash_enqueue(req, cryptd_hash_digest); 630} 631 632static int cryptd_hash_export(struct ahash_request *req, void *out) 633{ 634 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 635 636 return crypto_shash_export(&rctx->desc, out); 637} 638 639static int cryptd_hash_import(struct ahash_request *req, const void *in) 640{ 641 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 642 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 643 struct shash_desc *desc = cryptd_shash_desc(req); 644 645 desc->tfm = ctx->child; 646 647 return crypto_shash_import(desc, in); 648} 649 650static void cryptd_hash_free(struct ahash_instance *inst) 651{ 652 struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst); 653 654 crypto_drop_shash(&ctx->spawn); 655 kfree(inst); 656} 657 658static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, 659 struct crypto_attr_type *algt, 660 struct cryptd_queue *queue) 661{ 662 struct hashd_instance_ctx *ctx; 663 struct ahash_instance *inst; 664 struct shash_alg *alg; 665 u32 type; 666 u32 mask; 667 int err; 668 669 cryptd_type_and_mask(algt, &type, &mask); 670 671 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 672 if (!inst) 673 return -ENOMEM; 674 675 ctx = ahash_instance_ctx(inst); 676 ctx->queue = queue; 677 678 err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst), 679 crypto_attr_alg_name(tb[1]), type, mask); 680 if (err) 681 goto err_free_inst; 682 alg = crypto_spawn_shash_alg(&ctx->spawn); 683 684 err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base); 685 if (err) 686 goto err_free_inst; 687 688 inst->alg.halg.base.cra_flags |= CRYPTO_ALG_ASYNC | 689 (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL| 690 CRYPTO_ALG_OPTIONAL_KEY)); 691 inst->alg.halg.digestsize = alg->digestsize; 692 inst->alg.halg.statesize = alg->statesize; 693 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); 694 695 inst->alg.init_tfm = cryptd_hash_init_tfm; 696 inst->alg.clone_tfm = cryptd_hash_clone_tfm; 697 inst->alg.exit_tfm = cryptd_hash_exit_tfm; 698 699 inst->alg.init = cryptd_hash_init_enqueue; 700 inst->alg.update = cryptd_hash_update_enqueue; 701 inst->alg.final = cryptd_hash_final_enqueue; 702 inst->alg.finup = cryptd_hash_finup_enqueue; 703 inst->alg.export = cryptd_hash_export; 704 inst->alg.import = cryptd_hash_import; 705 if (crypto_shash_alg_has_setkey(alg)) 706 inst->alg.setkey = cryptd_hash_setkey; 707 inst->alg.digest = cryptd_hash_digest_enqueue; 708 709 inst->free = cryptd_hash_free; 710 711 err = ahash_register_instance(tmpl, inst); 712 if (err) { 713err_free_inst: 714 cryptd_hash_free(inst); 715 } 716 return err; 717} 718 719static int cryptd_aead_setkey(struct crypto_aead *parent, 720 const u8 *key, unsigned int keylen) 721{ 722 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); 723 struct crypto_aead *child = ctx->child; 724 725 return crypto_aead_setkey(child, key, keylen); 726} 727 728static int cryptd_aead_setauthsize(struct crypto_aead *parent, 729 unsigned int authsize) 730{ 731 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); 732 struct crypto_aead *child = ctx->child; 733 734 return crypto_aead_setauthsize(child, authsize); 735} 736 737static void cryptd_aead_crypt(struct aead_request *req, 738 struct crypto_aead *child, int err, 739 int (*crypt)(struct aead_request *req), 740 crypto_completion_t compl) 741{ 742 struct cryptd_aead_request_ctx *rctx; 743 struct aead_request *subreq; 744 struct cryptd_aead_ctx *ctx; 745 struct crypto_aead *tfm; 746 int refcnt; 747 748 rctx = aead_request_ctx(req); 749 subreq = &rctx->req; 750 req->base.complete = subreq->base.complete; 751 req->base.data = subreq->base.data; 752 753 tfm = crypto_aead_reqtfm(req); 754 755 if (unlikely(err == -EINPROGRESS)) 756 goto out; 757 758 aead_request_set_tfm(subreq, child); 759 aead_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, 760 NULL, NULL); 761 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 762 req->iv); 763 aead_request_set_ad(subreq, req->assoclen); 764 765 err = crypt(subreq); 766 767out: 768 ctx = crypto_aead_ctx(tfm); 769 refcnt = refcount_read(&ctx->refcnt); 770 771 local_bh_disable(); 772 aead_request_complete(req, err); 773 local_bh_enable(); 774 775 if (err == -EINPROGRESS) { 776 subreq->base.complete = req->base.complete; 777 subreq->base.data = req->base.data; 778 req->base.complete = compl; 779 req->base.data = req; 780 } else if (refcnt && refcount_dec_and_test(&ctx->refcnt)) 781 crypto_free_aead(tfm); 782} 783 784static void cryptd_aead_encrypt(void *data, int err) 785{ 786 struct aead_request *req = data; 787 struct cryptd_aead_ctx *ctx; 788 struct crypto_aead *child; 789 790 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 791 child = ctx->child; 792 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt, 793 cryptd_aead_encrypt); 794} 795 796static void cryptd_aead_decrypt(void *data, int err) 797{ 798 struct aead_request *req = data; 799 struct cryptd_aead_ctx *ctx; 800 struct crypto_aead *child; 801 802 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 803 child = ctx->child; 804 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt, 805 cryptd_aead_decrypt); 806} 807 808static int cryptd_aead_enqueue(struct aead_request *req, 809 crypto_completion_t compl) 810{ 811 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); 812 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 813 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); 814 struct aead_request *subreq = &rctx->req; 815 816 subreq->base.complete = req->base.complete; 817 subreq->base.data = req->base.data; 818 req->base.complete = compl; 819 req->base.data = req; 820 return cryptd_enqueue_request(queue, &req->base); 821} 822 823static int cryptd_aead_encrypt_enqueue(struct aead_request *req) 824{ 825 return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); 826} 827 828static int cryptd_aead_decrypt_enqueue(struct aead_request *req) 829{ 830 return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); 831} 832 833static int cryptd_aead_init_tfm(struct crypto_aead *tfm) 834{ 835 struct aead_instance *inst = aead_alg_instance(tfm); 836 struct aead_instance_ctx *ictx = aead_instance_ctx(inst); 837 struct crypto_aead_spawn *spawn = &ictx->aead_spawn; 838 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); 839 struct crypto_aead *cipher; 840 841 cipher = crypto_spawn_aead(spawn); 842 if (IS_ERR(cipher)) 843 return PTR_ERR(cipher); 844 845 ctx->child = cipher; 846 crypto_aead_set_reqsize( 847 tfm, sizeof(struct cryptd_aead_request_ctx) + 848 crypto_aead_reqsize(cipher)); 849 return 0; 850} 851 852static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) 853{ 854 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); 855 crypto_free_aead(ctx->child); 856} 857 858static void cryptd_aead_free(struct aead_instance *inst) 859{ 860 struct aead_instance_ctx *ctx = aead_instance_ctx(inst); 861 862 crypto_drop_aead(&ctx->aead_spawn); 863 kfree(inst); 864} 865 866static int cryptd_create_aead(struct crypto_template *tmpl, 867 struct rtattr **tb, 868 struct crypto_attr_type *algt, 869 struct cryptd_queue *queue) 870{ 871 struct aead_instance_ctx *ctx; 872 struct aead_instance *inst; 873 struct aead_alg *alg; 874 u32 type; 875 u32 mask; 876 int err; 877 878 cryptd_type_and_mask(algt, &type, &mask); 879 880 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 881 if (!inst) 882 return -ENOMEM; 883 884 ctx = aead_instance_ctx(inst); 885 ctx->queue = queue; 886 887 err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst), 888 crypto_attr_alg_name(tb[1]), type, mask); 889 if (err) 890 goto err_free_inst; 891 892 alg = crypto_spawn_aead_alg(&ctx->aead_spawn); 893 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); 894 if (err) 895 goto err_free_inst; 896 897 inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | 898 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); 899 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); 900 901 inst->alg.ivsize = crypto_aead_alg_ivsize(alg); 902 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 903 904 inst->alg.init = cryptd_aead_init_tfm; 905 inst->alg.exit = cryptd_aead_exit_tfm; 906 inst->alg.setkey = cryptd_aead_setkey; 907 inst->alg.setauthsize = cryptd_aead_setauthsize; 908 inst->alg.encrypt = cryptd_aead_encrypt_enqueue; 909 inst->alg.decrypt = cryptd_aead_decrypt_enqueue; 910 911 inst->free = cryptd_aead_free; 912 913 err = aead_register_instance(tmpl, inst); 914 if (err) { 915err_free_inst: 916 cryptd_aead_free(inst); 917 } 918 return err; 919} 920 921static struct cryptd_queue queue; 922 923static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) 924{ 925 struct crypto_attr_type *algt; 926 927 algt = crypto_get_attr_type(tb); 928 if (IS_ERR(algt)) 929 return PTR_ERR(algt); 930 931 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 932 case CRYPTO_ALG_TYPE_LSKCIPHER: 933 return cryptd_create_skcipher(tmpl, tb, algt, &queue); 934 case CRYPTO_ALG_TYPE_HASH: 935 return cryptd_create_hash(tmpl, tb, algt, &queue); 936 case CRYPTO_ALG_TYPE_AEAD: 937 return cryptd_create_aead(tmpl, tb, algt, &queue); 938 } 939 940 return -EINVAL; 941} 942 943static struct crypto_template cryptd_tmpl = { 944 .name = "cryptd", 945 .create = cryptd_create, 946 .module = THIS_MODULE, 947}; 948 949struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, 950 u32 type, u32 mask) 951{ 952 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 953 struct cryptd_skcipher_ctx *ctx; 954 struct crypto_skcipher *tfm; 955 956 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 957 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 958 return ERR_PTR(-EINVAL); 959 960 tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask); 961 if (IS_ERR(tfm)) 962 return ERR_CAST(tfm); 963 964 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 965 crypto_free_skcipher(tfm); 966 return ERR_PTR(-EINVAL); 967 } 968 969 ctx = crypto_skcipher_ctx(tfm); 970 refcount_set(&ctx->refcnt, 1); 971 972 return container_of(tfm, struct cryptd_skcipher, base); 973} 974EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher); 975 976struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm) 977{ 978 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); 979 980 return ctx->child; 981} 982EXPORT_SYMBOL_GPL(cryptd_skcipher_child); 983 984bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm) 985{ 986 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); 987 988 return refcount_read(&ctx->refcnt) - 1; 989} 990EXPORT_SYMBOL_GPL(cryptd_skcipher_queued); 991 992void cryptd_free_skcipher(struct cryptd_skcipher *tfm) 993{ 994 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); 995 996 if (refcount_dec_and_test(&ctx->refcnt)) 997 crypto_free_skcipher(&tfm->base); 998} 999EXPORT_SYMBOL_GPL(cryptd_free_skcipher); 1000 1001struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, 1002 u32 type, u32 mask) 1003{ 1004 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 1005 struct cryptd_hash_ctx *ctx; 1006 struct crypto_ahash *tfm; 1007 1008 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 1009 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 1010 return ERR_PTR(-EINVAL); 1011 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); 1012 if (IS_ERR(tfm)) 1013 return ERR_CAST(tfm); 1014 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 1015 crypto_free_ahash(tfm); 1016 return ERR_PTR(-EINVAL); 1017 } 1018 1019 ctx = crypto_ahash_ctx(tfm); 1020 refcount_set(&ctx->refcnt, 1); 1021 1022 return __cryptd_ahash_cast(tfm); 1023} 1024EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); 1025 1026struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) 1027{ 1028 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 1029 1030 return ctx->child; 1031} 1032EXPORT_SYMBOL_GPL(cryptd_ahash_child); 1033 1034struct shash_desc *cryptd_shash_desc(struct ahash_request *req) 1035{ 1036 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 1037 return &rctx->desc; 1038} 1039EXPORT_SYMBOL_GPL(cryptd_shash_desc); 1040 1041bool cryptd_ahash_queued(struct cryptd_ahash *tfm) 1042{ 1043 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 1044 1045 return refcount_read(&ctx->refcnt) - 1; 1046} 1047EXPORT_SYMBOL_GPL(cryptd_ahash_queued); 1048 1049void cryptd_free_ahash(struct cryptd_ahash *tfm) 1050{ 1051 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 1052 1053 if (refcount_dec_and_test(&ctx->refcnt)) 1054 crypto_free_ahash(&tfm->base); 1055} 1056EXPORT_SYMBOL_GPL(cryptd_free_ahash); 1057 1058struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, 1059 u32 type, u32 mask) 1060{ 1061 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 1062 struct cryptd_aead_ctx *ctx; 1063 struct crypto_aead *tfm; 1064 1065 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 1066 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 1067 return ERR_PTR(-EINVAL); 1068 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); 1069 if (IS_ERR(tfm)) 1070 return ERR_CAST(tfm); 1071 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 1072 crypto_free_aead(tfm); 1073 return ERR_PTR(-EINVAL); 1074 } 1075 1076 ctx = crypto_aead_ctx(tfm); 1077 refcount_set(&ctx->refcnt, 1); 1078 1079 return __cryptd_aead_cast(tfm); 1080} 1081EXPORT_SYMBOL_GPL(cryptd_alloc_aead); 1082 1083struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) 1084{ 1085 struct cryptd_aead_ctx *ctx; 1086 ctx = crypto_aead_ctx(&tfm->base); 1087 return ctx->child; 1088} 1089EXPORT_SYMBOL_GPL(cryptd_aead_child); 1090 1091bool cryptd_aead_queued(struct cryptd_aead *tfm) 1092{ 1093 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); 1094 1095 return refcount_read(&ctx->refcnt) - 1; 1096} 1097EXPORT_SYMBOL_GPL(cryptd_aead_queued); 1098 1099void cryptd_free_aead(struct cryptd_aead *tfm) 1100{ 1101 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); 1102 1103 if (refcount_dec_and_test(&ctx->refcnt)) 1104 crypto_free_aead(&tfm->base); 1105} 1106EXPORT_SYMBOL_GPL(cryptd_free_aead); 1107 1108static int __init cryptd_init(void) 1109{ 1110 int err; 1111 1112 cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1113 1); 1114 if (!cryptd_wq) 1115 return -ENOMEM; 1116 1117 err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); 1118 if (err) 1119 goto err_destroy_wq; 1120 1121 err = crypto_register_template(&cryptd_tmpl); 1122 if (err) 1123 goto err_fini_queue; 1124 1125 return 0; 1126 1127err_fini_queue: 1128 cryptd_fini_queue(&queue); 1129err_destroy_wq: 1130 destroy_workqueue(cryptd_wq); 1131 return err; 1132} 1133 1134static void __exit cryptd_exit(void) 1135{ 1136 destroy_workqueue(cryptd_wq); 1137 cryptd_fini_queue(&queue); 1138 crypto_unregister_template(&cryptd_tmpl); 1139} 1140 1141subsys_initcall(cryptd_init); 1142module_exit(cryptd_exit); 1143 1144MODULE_LICENSE("GPL"); 1145MODULE_DESCRIPTION("Software async crypto daemon"); 1146MODULE_ALIAS_CRYPTO("cryptd"); 1147