1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. 6 * 7 * Author: Tom Lendacky <thomas.lendacky@amd.com> 8 * Author: Gary R Hook <gary.hook@amd.com> 9 */ 10 11#include <linux/dma-mapping.h> 12#include <linux/module.h> 13#include <linux/kernel.h> 14#include <linux/interrupt.h> 15#include <crypto/scatterwalk.h> 16#include <crypto/des.h> 17#include <linux/ccp.h> 18 19#include "ccp-dev.h" 20 21/* SHA initial context values */ 22static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = { 23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), 24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), 25 cpu_to_be32(SHA1_H4), 26}; 27 28static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), 30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), 31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), 32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), 33}; 34 35static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = { 36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), 37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), 38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), 39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), 40}; 41 42static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), 44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), 45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), 46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7), 47}; 48 49static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = { 50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), 51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), 52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), 53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7), 54}; 55 56#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \ 57 ccp_gen_jobid(ccp) : 0) 58 59static u32 ccp_gen_jobid(struct ccp_device *ccp) 60{ 61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; 62} 63 64static void ccp_sg_free(struct ccp_sg_workarea *wa) 65{ 66 if (wa->dma_count) 67 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir); 68 69 wa->dma_count = 0; 70} 71 72static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, 73 struct scatterlist *sg, u64 len, 74 enum dma_data_direction dma_dir) 75{ 76 memset(wa, 0, sizeof(*wa)); 77 78 wa->sg = sg; 79 if (!sg) 80 return 0; 81 82 wa->nents = sg_nents_for_len(sg, len); 83 if (wa->nents < 0) 84 return wa->nents; 85 86 wa->bytes_left = len; 87 wa->sg_used = 0; 88 89 if (len == 0) 90 return 0; 91 92 if (dma_dir == DMA_NONE) 93 return 0; 94 95 wa->dma_sg = sg; 96 wa->dma_sg_head = sg; 97 wa->dma_dev = dev; 98 wa->dma_dir = dma_dir; 99 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); 100 if (!wa->dma_count) 101 return -ENOMEM; 102 103 return 0; 104} 105 106static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) 107{ 108 unsigned int nbytes = min_t(u64, len, wa->bytes_left); 109 unsigned int sg_combined_len = 0; 110 111 if (!wa->sg) 112 return; 113 114 wa->sg_used += nbytes; 115 wa->bytes_left -= nbytes; 116 if (wa->sg_used == sg_dma_len(wa->dma_sg)) { 117 /* Advance to the next DMA scatterlist entry */ 118 wa->dma_sg = sg_next(wa->dma_sg); 119 120 /* In the case that the DMA mapped scatterlist has entries 121 * that have been merged, the non-DMA mapped scatterlist 122 * must be advanced multiple times for each merged entry. 123 * This ensures that the current non-DMA mapped entry 124 * corresponds to the current DMA mapped entry. 125 */ 126 do { 127 sg_combined_len += wa->sg->length; 128 wa->sg = sg_next(wa->sg); 129 } while (wa->sg_used > sg_combined_len); 130 131 wa->sg_used = 0; 132 } 133} 134 135static void ccp_dm_free(struct ccp_dm_workarea *wa) 136{ 137 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { 138 if (wa->address) 139 dma_pool_free(wa->dma_pool, wa->address, 140 wa->dma.address); 141 } else { 142 if (wa->dma.address) 143 dma_unmap_single(wa->dev, wa->dma.address, wa->length, 144 wa->dma.dir); 145 kfree(wa->address); 146 } 147 148 wa->address = NULL; 149 wa->dma.address = 0; 150} 151 152static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, 153 struct ccp_cmd_queue *cmd_q, 154 unsigned int len, 155 enum dma_data_direction dir) 156{ 157 memset(wa, 0, sizeof(*wa)); 158 159 if (!len) 160 return 0; 161 162 wa->dev = cmd_q->ccp->dev; 163 wa->length = len; 164 165 if (len <= CCP_DMAPOOL_MAX_SIZE) { 166 wa->dma_pool = cmd_q->dma_pool; 167 168 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL, 169 &wa->dma.address); 170 if (!wa->address) 171 return -ENOMEM; 172 173 wa->dma.length = CCP_DMAPOOL_MAX_SIZE; 174 175 } else { 176 wa->address = kzalloc(len, GFP_KERNEL); 177 if (!wa->address) 178 return -ENOMEM; 179 180 wa->dma.address = dma_map_single(wa->dev, wa->address, len, 181 dir); 182 if (dma_mapping_error(wa->dev, wa->dma.address)) { 183 kfree(wa->address); 184 wa->address = NULL; 185 return -ENOMEM; 186 } 187 188 wa->dma.length = len; 189 } 190 wa->dma.dir = dir; 191 192 return 0; 193} 194 195static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 196 struct scatterlist *sg, unsigned int sg_offset, 197 unsigned int len) 198{ 199 WARN_ON(!wa->address); 200 201 if (len > (wa->length - wa_offset)) 202 return -EINVAL; 203 204 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 205 0); 206 return 0; 207} 208 209static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 210 struct scatterlist *sg, unsigned int sg_offset, 211 unsigned int len) 212{ 213 WARN_ON(!wa->address); 214 215 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, 216 1); 217} 218 219static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, 220 unsigned int wa_offset, 221 struct scatterlist *sg, 222 unsigned int sg_offset, 223 unsigned int len) 224{ 225 u8 *p, *q; 226 int rc; 227 228 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len); 229 if (rc) 230 return rc; 231 232 p = wa->address + wa_offset; 233 q = p + len - 1; 234 while (p < q) { 235 *p = *p ^ *q; 236 *q = *p ^ *q; 237 *p = *p ^ *q; 238 p++; 239 q--; 240 } 241 return 0; 242} 243 244static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, 245 unsigned int wa_offset, 246 struct scatterlist *sg, 247 unsigned int sg_offset, 248 unsigned int len) 249{ 250 u8 *p, *q; 251 252 p = wa->address + wa_offset; 253 q = p + len - 1; 254 while (p < q) { 255 *p = *p ^ *q; 256 *q = *p ^ *q; 257 *p = *p ^ *q; 258 p++; 259 q--; 260 } 261 262 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len); 263} 264 265static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) 266{ 267 ccp_dm_free(&data->dm_wa); 268 ccp_sg_free(&data->sg_wa); 269} 270 271static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, 272 struct scatterlist *sg, u64 sg_len, 273 unsigned int dm_len, 274 enum dma_data_direction dir) 275{ 276 int ret; 277 278 memset(data, 0, sizeof(*data)); 279 280 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, 281 dir); 282 if (ret) 283 goto e_err; 284 285 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); 286 if (ret) 287 goto e_err; 288 289 return 0; 290 291e_err: 292 ccp_free_data(data, cmd_q); 293 294 return ret; 295} 296 297static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) 298{ 299 struct ccp_sg_workarea *sg_wa = &data->sg_wa; 300 struct ccp_dm_workarea *dm_wa = &data->dm_wa; 301 unsigned int buf_count, nbytes; 302 303 /* Clear the buffer if setting it */ 304 if (!from) 305 memset(dm_wa->address, 0, dm_wa->length); 306 307 if (!sg_wa->sg) 308 return 0; 309 310 /* Perform the copy operation 311 * nbytes will always be <= UINT_MAX because dm_wa->length is 312 * an unsigned int 313 */ 314 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); 315 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, 316 nbytes, from); 317 318 /* Update the structures and generate the count */ 319 buf_count = 0; 320 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { 321 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used, 322 dm_wa->length - buf_count); 323 nbytes = min_t(u64, sg_wa->bytes_left, nbytes); 324 325 buf_count += nbytes; 326 ccp_update_sg_workarea(sg_wa, nbytes); 327 } 328 329 return buf_count; 330} 331 332static unsigned int ccp_fill_queue_buf(struct ccp_data *data) 333{ 334 return ccp_queue_buf(data, 0); 335} 336 337static unsigned int ccp_empty_queue_buf(struct ccp_data *data) 338{ 339 return ccp_queue_buf(data, 1); 340} 341 342static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, 343 struct ccp_op *op, unsigned int block_size, 344 bool blocksize_op) 345{ 346 unsigned int sg_src_len, sg_dst_len, op_len; 347 348 /* The CCP can only DMA from/to one address each per operation. This 349 * requires that we find the smallest DMA area between the source 350 * and destination. The resulting len values will always be <= UINT_MAX 351 * because the dma length is an unsigned int. 352 */ 353 sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used; 354 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); 355 356 if (dst) { 357 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used; 358 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 359 op_len = min(sg_src_len, sg_dst_len); 360 } else { 361 op_len = sg_src_len; 362 } 363 364 /* The data operation length will be at least block_size in length 365 * or the smaller of available sg room remaining for the source or 366 * the destination 367 */ 368 op_len = max(op_len, block_size); 369 370 /* Unless we have to buffer data, there's no reason to wait */ 371 op->soc = 0; 372 373 if (sg_src_len < block_size) { 374 /* Not enough data in the sg element, so it 375 * needs to be buffered into a blocksize chunk 376 */ 377 int cp_len = ccp_fill_queue_buf(src); 378 379 op->soc = 1; 380 op->src.u.dma.address = src->dm_wa.dma.address; 381 op->src.u.dma.offset = 0; 382 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; 383 } else { 384 /* Enough data in the sg element, but we need to 385 * adjust for any previously copied data 386 */ 387 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg); 388 op->src.u.dma.offset = src->sg_wa.sg_used; 389 op->src.u.dma.length = op_len & ~(block_size - 1); 390 391 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); 392 } 393 394 if (dst) { 395 if (sg_dst_len < block_size) { 396 /* Not enough room in the sg element or we're on the 397 * last piece of data (when using padding), so the 398 * output needs to be buffered into a blocksize chunk 399 */ 400 op->soc = 1; 401 op->dst.u.dma.address = dst->dm_wa.dma.address; 402 op->dst.u.dma.offset = 0; 403 op->dst.u.dma.length = op->src.u.dma.length; 404 } else { 405 /* Enough room in the sg element, but we need to 406 * adjust for any previously used area 407 */ 408 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg); 409 op->dst.u.dma.offset = dst->sg_wa.sg_used; 410 op->dst.u.dma.length = op->src.u.dma.length; 411 } 412 } 413} 414 415static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, 416 struct ccp_op *op) 417{ 418 op->init = 0; 419 420 if (dst) { 421 if (op->dst.u.dma.address == dst->dm_wa.dma.address) 422 ccp_empty_queue_buf(dst); 423 else 424 ccp_update_sg_workarea(&dst->sg_wa, 425 op->dst.u.dma.length); 426 } 427} 428 429static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q, 430 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 431 u32 byte_swap, bool from) 432{ 433 struct ccp_op op; 434 435 memset(&op, 0, sizeof(op)); 436 437 op.cmd_q = cmd_q; 438 op.jobid = jobid; 439 op.eom = 1; 440 441 if (from) { 442 op.soc = 1; 443 op.src.type = CCP_MEMTYPE_SB; 444 op.src.u.sb = sb; 445 op.dst.type = CCP_MEMTYPE_SYSTEM; 446 op.dst.u.dma.address = wa->dma.address; 447 op.dst.u.dma.length = wa->length; 448 } else { 449 op.src.type = CCP_MEMTYPE_SYSTEM; 450 op.src.u.dma.address = wa->dma.address; 451 op.src.u.dma.length = wa->length; 452 op.dst.type = CCP_MEMTYPE_SB; 453 op.dst.u.sb = sb; 454 } 455 456 op.u.passthru.byte_swap = byte_swap; 457 458 return cmd_q->ccp->vdata->perform->passthru(&op); 459} 460 461static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q, 462 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 463 u32 byte_swap) 464{ 465 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false); 466} 467 468static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q, 469 struct ccp_dm_workarea *wa, u32 jobid, u32 sb, 470 u32 byte_swap) 471{ 472 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true); 473} 474 475static noinline_for_stack int 476ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 477{ 478 struct ccp_aes_engine *aes = &cmd->u.aes; 479 struct ccp_dm_workarea key, ctx; 480 struct ccp_data src; 481 struct ccp_op op; 482 unsigned int dm_offset; 483 int ret; 484 485 if (!((aes->key_len == AES_KEYSIZE_128) || 486 (aes->key_len == AES_KEYSIZE_192) || 487 (aes->key_len == AES_KEYSIZE_256))) 488 return -EINVAL; 489 490 if (aes->src_len & (AES_BLOCK_SIZE - 1)) 491 return -EINVAL; 492 493 if (aes->iv_len != AES_BLOCK_SIZE) 494 return -EINVAL; 495 496 if (!aes->key || !aes->iv || !aes->src) 497 return -EINVAL; 498 499 if (aes->cmac_final) { 500 if (aes->cmac_key_len != AES_BLOCK_SIZE) 501 return -EINVAL; 502 503 if (!aes->cmac_key) 504 return -EINVAL; 505 } 506 507 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 508 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 509 510 ret = -EIO; 511 memset(&op, 0, sizeof(op)); 512 op.cmd_q = cmd_q; 513 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 514 op.sb_key = cmd_q->sb_key; 515 op.sb_ctx = cmd_q->sb_ctx; 516 op.init = 1; 517 op.u.aes.type = aes->type; 518 op.u.aes.mode = aes->mode; 519 op.u.aes.action = aes->action; 520 521 /* All supported key sizes fit in a single (32-byte) SB entry 522 * and must be in little endian format. Use the 256-bit byte 523 * swap passthru option to convert from big endian to little 524 * endian. 525 */ 526 ret = ccp_init_dm_workarea(&key, cmd_q, 527 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 528 DMA_TO_DEVICE); 529 if (ret) 530 return ret; 531 532 dm_offset = CCP_SB_BYTES - aes->key_len; 533 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 534 if (ret) 535 goto e_key; 536 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 537 CCP_PASSTHRU_BYTESWAP_256BIT); 538 if (ret) { 539 cmd->engine_error = cmd_q->cmd_error; 540 goto e_key; 541 } 542 543 /* The AES context fits in a single (32-byte) SB entry and 544 * must be in little endian format. Use the 256-bit byte swap 545 * passthru option to convert from big endian to little endian. 546 */ 547 ret = ccp_init_dm_workarea(&ctx, cmd_q, 548 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 549 DMA_BIDIRECTIONAL); 550 if (ret) 551 goto e_key; 552 553 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 554 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 555 if (ret) 556 goto e_ctx; 557 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 558 CCP_PASSTHRU_BYTESWAP_256BIT); 559 if (ret) { 560 cmd->engine_error = cmd_q->cmd_error; 561 goto e_ctx; 562 } 563 564 /* Send data to the CCP AES engine */ 565 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 566 AES_BLOCK_SIZE, DMA_TO_DEVICE); 567 if (ret) 568 goto e_ctx; 569 570 while (src.sg_wa.bytes_left) { 571 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); 572 if (aes->cmac_final && !src.sg_wa.bytes_left) { 573 op.eom = 1; 574 575 /* Push the K1/K2 key to the CCP now */ 576 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, 577 op.sb_ctx, 578 CCP_PASSTHRU_BYTESWAP_256BIT); 579 if (ret) { 580 cmd->engine_error = cmd_q->cmd_error; 581 goto e_src; 582 } 583 584 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, 585 aes->cmac_key_len); 586 if (ret) 587 goto e_src; 588 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 589 CCP_PASSTHRU_BYTESWAP_256BIT); 590 if (ret) { 591 cmd->engine_error = cmd_q->cmd_error; 592 goto e_src; 593 } 594 } 595 596 ret = cmd_q->ccp->vdata->perform->aes(&op); 597 if (ret) { 598 cmd->engine_error = cmd_q->cmd_error; 599 goto e_src; 600 } 601 602 ccp_process_data(&src, NULL, &op); 603 } 604 605 /* Retrieve the AES context - convert from LE to BE using 606 * 32-byte (256-bit) byteswapping 607 */ 608 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 609 CCP_PASSTHRU_BYTESWAP_256BIT); 610 if (ret) { 611 cmd->engine_error = cmd_q->cmd_error; 612 goto e_src; 613 } 614 615 /* ...but we only need AES_BLOCK_SIZE bytes */ 616 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 617 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 618 619e_src: 620 ccp_free_data(&src, cmd_q); 621 622e_ctx: 623 ccp_dm_free(&ctx); 624 625e_key: 626 ccp_dm_free(&key); 627 628 return ret; 629} 630 631static noinline_for_stack int 632ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 633{ 634 struct ccp_aes_engine *aes = &cmd->u.aes; 635 struct ccp_dm_workarea key, ctx, final_wa, tag; 636 struct ccp_data src, dst; 637 struct ccp_data aad; 638 struct ccp_op op; 639 unsigned int dm_offset; 640 unsigned int authsize; 641 unsigned int jobid; 642 unsigned int ilen; 643 bool in_place = true; /* Default value */ 644 __be64 *final; 645 int ret; 646 647 struct scatterlist *p_inp, sg_inp[2]; 648 struct scatterlist *p_tag, sg_tag[2]; 649 struct scatterlist *p_outp, sg_outp[2]; 650 struct scatterlist *p_aad; 651 652 if (!aes->iv) 653 return -EINVAL; 654 655 if (!((aes->key_len == AES_KEYSIZE_128) || 656 (aes->key_len == AES_KEYSIZE_192) || 657 (aes->key_len == AES_KEYSIZE_256))) 658 return -EINVAL; 659 660 if (!aes->key) /* Gotta have a key SGL */ 661 return -EINVAL; 662 663 /* Zero defaults to 16 bytes, the maximum size */ 664 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE; 665 switch (authsize) { 666 case 16: 667 case 15: 668 case 14: 669 case 13: 670 case 12: 671 case 8: 672 case 4: 673 break; 674 default: 675 return -EINVAL; 676 } 677 678 /* First, decompose the source buffer into AAD & PT, 679 * and the destination buffer into AAD, CT & tag, or 680 * the input into CT & tag. 681 * It is expected that the input and output SGs will 682 * be valid, even if the AAD and input lengths are 0. 683 */ 684 p_aad = aes->src; 685 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len); 686 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len); 687 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 688 ilen = aes->src_len; 689 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen); 690 } else { 691 /* Input length for decryption includes tag */ 692 ilen = aes->src_len - authsize; 693 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen); 694 } 695 696 jobid = CCP_NEW_JOBID(cmd_q->ccp); 697 698 memset(&op, 0, sizeof(op)); 699 op.cmd_q = cmd_q; 700 op.jobid = jobid; 701 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 702 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 703 op.init = 1; 704 op.u.aes.type = aes->type; 705 706 /* Copy the key to the LSB */ 707 ret = ccp_init_dm_workarea(&key, cmd_q, 708 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 709 DMA_TO_DEVICE); 710 if (ret) 711 return ret; 712 713 dm_offset = CCP_SB_BYTES - aes->key_len; 714 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 715 if (ret) 716 goto e_key; 717 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 718 CCP_PASSTHRU_BYTESWAP_256BIT); 719 if (ret) { 720 cmd->engine_error = cmd_q->cmd_error; 721 goto e_key; 722 } 723 724 /* Copy the context (IV) to the LSB. 725 * There is an assumption here that the IV is 96 bits in length, plus 726 * a nonce of 32 bits. If no IV is present, use a zeroed buffer. 727 */ 728 ret = ccp_init_dm_workarea(&ctx, cmd_q, 729 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 730 DMA_BIDIRECTIONAL); 731 if (ret) 732 goto e_key; 733 734 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len; 735 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 736 if (ret) 737 goto e_ctx; 738 739 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 740 CCP_PASSTHRU_BYTESWAP_256BIT); 741 if (ret) { 742 cmd->engine_error = cmd_q->cmd_error; 743 goto e_ctx; 744 } 745 746 op.init = 1; 747 if (aes->aad_len > 0) { 748 /* Step 1: Run a GHASH over the Additional Authenticated Data */ 749 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len, 750 AES_BLOCK_SIZE, 751 DMA_TO_DEVICE); 752 if (ret) 753 goto e_ctx; 754 755 op.u.aes.mode = CCP_AES_MODE_GHASH; 756 op.u.aes.action = CCP_AES_GHASHAAD; 757 758 while (aad.sg_wa.bytes_left) { 759 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true); 760 761 ret = cmd_q->ccp->vdata->perform->aes(&op); 762 if (ret) { 763 cmd->engine_error = cmd_q->cmd_error; 764 goto e_aad; 765 } 766 767 ccp_process_data(&aad, NULL, &op); 768 op.init = 0; 769 } 770 } 771 772 op.u.aes.mode = CCP_AES_MODE_GCTR; 773 op.u.aes.action = aes->action; 774 775 if (ilen > 0) { 776 /* Step 2: Run a GCTR over the plaintext */ 777 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false; 778 779 ret = ccp_init_data(&src, cmd_q, p_inp, ilen, 780 AES_BLOCK_SIZE, 781 in_place ? DMA_BIDIRECTIONAL 782 : DMA_TO_DEVICE); 783 if (ret) 784 goto e_aad; 785 786 if (in_place) { 787 dst = src; 788 } else { 789 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen, 790 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 791 if (ret) 792 goto e_src; 793 } 794 795 op.soc = 0; 796 op.eom = 0; 797 op.init = 1; 798 while (src.sg_wa.bytes_left) { 799 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 800 if (!src.sg_wa.bytes_left) { 801 unsigned int nbytes = ilen % AES_BLOCK_SIZE; 802 803 if (nbytes) { 804 op.eom = 1; 805 op.u.aes.size = (nbytes * 8) - 1; 806 } 807 } 808 809 ret = cmd_q->ccp->vdata->perform->aes(&op); 810 if (ret) { 811 cmd->engine_error = cmd_q->cmd_error; 812 goto e_dst; 813 } 814 815 ccp_process_data(&src, &dst, &op); 816 op.init = 0; 817 } 818 } 819 820 /* Step 3: Update the IV portion of the context with the original IV */ 821 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 822 CCP_PASSTHRU_BYTESWAP_256BIT); 823 if (ret) { 824 cmd->engine_error = cmd_q->cmd_error; 825 goto e_dst; 826 } 827 828 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 829 if (ret) 830 goto e_dst; 831 832 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 833 CCP_PASSTHRU_BYTESWAP_256BIT); 834 if (ret) { 835 cmd->engine_error = cmd_q->cmd_error; 836 goto e_dst; 837 } 838 839 /* Step 4: Concatenate the lengths of the AAD and source, and 840 * hash that 16 byte buffer. 841 */ 842 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE, 843 DMA_BIDIRECTIONAL); 844 if (ret) 845 goto e_dst; 846 final = (__be64 *)final_wa.address; 847 final[0] = cpu_to_be64(aes->aad_len * 8); 848 final[1] = cpu_to_be64(ilen * 8); 849 850 memset(&op, 0, sizeof(op)); 851 op.cmd_q = cmd_q; 852 op.jobid = jobid; 853 op.sb_key = cmd_q->sb_key; /* Pre-allocated */ 854 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 855 op.init = 1; 856 op.u.aes.type = aes->type; 857 op.u.aes.mode = CCP_AES_MODE_GHASH; 858 op.u.aes.action = CCP_AES_GHASHFINAL; 859 op.src.type = CCP_MEMTYPE_SYSTEM; 860 op.src.u.dma.address = final_wa.dma.address; 861 op.src.u.dma.length = AES_BLOCK_SIZE; 862 op.dst.type = CCP_MEMTYPE_SYSTEM; 863 op.dst.u.dma.address = final_wa.dma.address; 864 op.dst.u.dma.length = AES_BLOCK_SIZE; 865 op.eom = 1; 866 op.u.aes.size = 0; 867 ret = cmd_q->ccp->vdata->perform->aes(&op); 868 if (ret) 869 goto e_final_wa; 870 871 if (aes->action == CCP_AES_ACTION_ENCRYPT) { 872 /* Put the ciphered tag after the ciphertext. */ 873 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize); 874 } else { 875 /* Does this ciphered tag match the input? */ 876 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize, 877 DMA_BIDIRECTIONAL); 878 if (ret) 879 goto e_final_wa; 880 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize); 881 if (ret) { 882 ccp_dm_free(&tag); 883 goto e_final_wa; 884 } 885 886 ret = crypto_memneq(tag.address, final_wa.address, 887 authsize) ? -EBADMSG : 0; 888 ccp_dm_free(&tag); 889 } 890 891e_final_wa: 892 ccp_dm_free(&final_wa); 893 894e_dst: 895 if (ilen > 0 && !in_place) 896 ccp_free_data(&dst, cmd_q); 897 898e_src: 899 if (ilen > 0) 900 ccp_free_data(&src, cmd_q); 901 902e_aad: 903 if (aes->aad_len) 904 ccp_free_data(&aad, cmd_q); 905 906e_ctx: 907 ccp_dm_free(&ctx); 908 909e_key: 910 ccp_dm_free(&key); 911 912 return ret; 913} 914 915static noinline_for_stack int 916ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 917{ 918 struct ccp_aes_engine *aes = &cmd->u.aes; 919 struct ccp_dm_workarea key, ctx; 920 struct ccp_data src, dst; 921 struct ccp_op op; 922 unsigned int dm_offset; 923 bool in_place = false; 924 int ret; 925 926 if (!((aes->key_len == AES_KEYSIZE_128) || 927 (aes->key_len == AES_KEYSIZE_192) || 928 (aes->key_len == AES_KEYSIZE_256))) 929 return -EINVAL; 930 931 if (((aes->mode == CCP_AES_MODE_ECB) || 932 (aes->mode == CCP_AES_MODE_CBC)) && 933 (aes->src_len & (AES_BLOCK_SIZE - 1))) 934 return -EINVAL; 935 936 if (!aes->key || !aes->src || !aes->dst) 937 return -EINVAL; 938 939 if (aes->mode != CCP_AES_MODE_ECB) { 940 if (aes->iv_len != AES_BLOCK_SIZE) 941 return -EINVAL; 942 943 if (!aes->iv) 944 return -EINVAL; 945 } 946 947 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1); 948 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1); 949 950 ret = -EIO; 951 memset(&op, 0, sizeof(op)); 952 op.cmd_q = cmd_q; 953 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 954 op.sb_key = cmd_q->sb_key; 955 op.sb_ctx = cmd_q->sb_ctx; 956 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; 957 op.u.aes.type = aes->type; 958 op.u.aes.mode = aes->mode; 959 op.u.aes.action = aes->action; 960 961 /* All supported key sizes fit in a single (32-byte) SB entry 962 * and must be in little endian format. Use the 256-bit byte 963 * swap passthru option to convert from big endian to little 964 * endian. 965 */ 966 ret = ccp_init_dm_workarea(&key, cmd_q, 967 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES, 968 DMA_TO_DEVICE); 969 if (ret) 970 return ret; 971 972 dm_offset = CCP_SB_BYTES - aes->key_len; 973 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); 974 if (ret) 975 goto e_key; 976 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 977 CCP_PASSTHRU_BYTESWAP_256BIT); 978 if (ret) { 979 cmd->engine_error = cmd_q->cmd_error; 980 goto e_key; 981 } 982 983 /* The AES context fits in a single (32-byte) SB entry and 984 * must be in little endian format. Use the 256-bit byte swap 985 * passthru option to convert from big endian to little endian. 986 */ 987 ret = ccp_init_dm_workarea(&ctx, cmd_q, 988 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES, 989 DMA_BIDIRECTIONAL); 990 if (ret) 991 goto e_key; 992 993 if (aes->mode != CCP_AES_MODE_ECB) { 994 /* Load the AES context - convert to LE */ 995 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 996 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 997 if (ret) 998 goto e_ctx; 999 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1000 CCP_PASSTHRU_BYTESWAP_256BIT); 1001 if (ret) { 1002 cmd->engine_error = cmd_q->cmd_error; 1003 goto e_ctx; 1004 } 1005 } 1006 switch (aes->mode) { 1007 case CCP_AES_MODE_CFB: /* CFB128 only */ 1008 case CCP_AES_MODE_CTR: 1009 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1; 1010 break; 1011 default: 1012 op.u.aes.size = 0; 1013 } 1014 1015 /* Prepare the input and output data workareas. For in-place 1016 * operations we need to set the dma direction to BIDIRECTIONAL 1017 * and copy the src workarea to the dst workarea. 1018 */ 1019 if (sg_virt(aes->src) == sg_virt(aes->dst)) 1020 in_place = true; 1021 1022 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, 1023 AES_BLOCK_SIZE, 1024 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1025 if (ret) 1026 goto e_ctx; 1027 1028 if (in_place) { 1029 dst = src; 1030 } else { 1031 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1032 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1033 if (ret) 1034 goto e_src; 1035 } 1036 1037 /* Send data to the CCP AES engine */ 1038 while (src.sg_wa.bytes_left) { 1039 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); 1040 if (!src.sg_wa.bytes_left) { 1041 op.eom = 1; 1042 1043 /* Since we don't retrieve the AES context in ECB 1044 * mode we have to wait for the operation to complete 1045 * on the last piece of data 1046 */ 1047 if (aes->mode == CCP_AES_MODE_ECB) 1048 op.soc = 1; 1049 } 1050 1051 ret = cmd_q->ccp->vdata->perform->aes(&op); 1052 if (ret) { 1053 cmd->engine_error = cmd_q->cmd_error; 1054 goto e_dst; 1055 } 1056 1057 ccp_process_data(&src, &dst, &op); 1058 } 1059 1060 if (aes->mode != CCP_AES_MODE_ECB) { 1061 /* Retrieve the AES context - convert from LE to BE using 1062 * 32-byte (256-bit) byteswapping 1063 */ 1064 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1065 CCP_PASSTHRU_BYTESWAP_256BIT); 1066 if (ret) { 1067 cmd->engine_error = cmd_q->cmd_error; 1068 goto e_dst; 1069 } 1070 1071 /* ...but we only need AES_BLOCK_SIZE bytes */ 1072 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1073 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); 1074 } 1075 1076e_dst: 1077 if (!in_place) 1078 ccp_free_data(&dst, cmd_q); 1079 1080e_src: 1081 ccp_free_data(&src, cmd_q); 1082 1083e_ctx: 1084 ccp_dm_free(&ctx); 1085 1086e_key: 1087 ccp_dm_free(&key); 1088 1089 return ret; 1090} 1091 1092static noinline_for_stack int 1093ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1094{ 1095 struct ccp_xts_aes_engine *xts = &cmd->u.xts; 1096 struct ccp_dm_workarea key, ctx; 1097 struct ccp_data src, dst; 1098 struct ccp_op op; 1099 unsigned int unit_size, dm_offset; 1100 bool in_place = false; 1101 unsigned int sb_count; 1102 enum ccp_aes_type aestype; 1103 int ret; 1104 1105 switch (xts->unit_size) { 1106 case CCP_XTS_AES_UNIT_SIZE_16: 1107 unit_size = 16; 1108 break; 1109 case CCP_XTS_AES_UNIT_SIZE_512: 1110 unit_size = 512; 1111 break; 1112 case CCP_XTS_AES_UNIT_SIZE_1024: 1113 unit_size = 1024; 1114 break; 1115 case CCP_XTS_AES_UNIT_SIZE_2048: 1116 unit_size = 2048; 1117 break; 1118 case CCP_XTS_AES_UNIT_SIZE_4096: 1119 unit_size = 4096; 1120 break; 1121 1122 default: 1123 return -EINVAL; 1124 } 1125 1126 if (xts->key_len == AES_KEYSIZE_128) 1127 aestype = CCP_AES_TYPE_128; 1128 else if (xts->key_len == AES_KEYSIZE_256) 1129 aestype = CCP_AES_TYPE_256; 1130 else 1131 return -EINVAL; 1132 1133 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) 1134 return -EINVAL; 1135 1136 if (xts->iv_len != AES_BLOCK_SIZE) 1137 return -EINVAL; 1138 1139 if (!xts->key || !xts->iv || !xts->src || !xts->dst) 1140 return -EINVAL; 1141 1142 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1); 1143 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1); 1144 1145 ret = -EIO; 1146 memset(&op, 0, sizeof(op)); 1147 op.cmd_q = cmd_q; 1148 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1149 op.sb_key = cmd_q->sb_key; 1150 op.sb_ctx = cmd_q->sb_ctx; 1151 op.init = 1; 1152 op.u.xts.type = aestype; 1153 op.u.xts.action = xts->action; 1154 op.u.xts.unit_size = xts->unit_size; 1155 1156 /* A version 3 device only supports 128-bit keys, which fits into a 1157 * single SB entry. A version 5 device uses a 512-bit vector, so two 1158 * SB entries. 1159 */ 1160 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) 1161 sb_count = CCP_XTS_AES_KEY_SB_COUNT; 1162 else 1163 sb_count = CCP5_XTS_AES_KEY_SB_COUNT; 1164 ret = ccp_init_dm_workarea(&key, cmd_q, 1165 sb_count * CCP_SB_BYTES, 1166 DMA_TO_DEVICE); 1167 if (ret) 1168 return ret; 1169 1170 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1171 /* All supported key sizes must be in little endian format. 1172 * Use the 256-bit byte swap passthru option to convert from 1173 * big endian to little endian. 1174 */ 1175 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; 1176 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); 1177 if (ret) 1178 goto e_key; 1179 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len); 1180 if (ret) 1181 goto e_key; 1182 } else { 1183 /* Version 5 CCPs use a 512-bit space for the key: each portion 1184 * occupies 256 bits, or one entire slot, and is zero-padded. 1185 */ 1186 unsigned int pad; 1187 1188 dm_offset = CCP_SB_BYTES; 1189 pad = dm_offset - xts->key_len; 1190 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len); 1191 if (ret) 1192 goto e_key; 1193 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key, 1194 xts->key_len, xts->key_len); 1195 if (ret) 1196 goto e_key; 1197 } 1198 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1199 CCP_PASSTHRU_BYTESWAP_256BIT); 1200 if (ret) { 1201 cmd->engine_error = cmd_q->cmd_error; 1202 goto e_key; 1203 } 1204 1205 /* The AES context fits in a single (32-byte) SB entry and 1206 * for XTS is already in little endian format so no byte swapping 1207 * is needed. 1208 */ 1209 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1210 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES, 1211 DMA_BIDIRECTIONAL); 1212 if (ret) 1213 goto e_key; 1214 1215 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); 1216 if (ret) 1217 goto e_ctx; 1218 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1219 CCP_PASSTHRU_BYTESWAP_NOOP); 1220 if (ret) { 1221 cmd->engine_error = cmd_q->cmd_error; 1222 goto e_ctx; 1223 } 1224 1225 /* Prepare the input and output data workareas. For in-place 1226 * operations we need to set the dma direction to BIDIRECTIONAL 1227 * and copy the src workarea to the dst workarea. 1228 */ 1229 if (sg_virt(xts->src) == sg_virt(xts->dst)) 1230 in_place = true; 1231 1232 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, 1233 unit_size, 1234 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1235 if (ret) 1236 goto e_ctx; 1237 1238 if (in_place) { 1239 dst = src; 1240 } else { 1241 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1242 unit_size, DMA_FROM_DEVICE); 1243 if (ret) 1244 goto e_src; 1245 } 1246 1247 /* Send data to the CCP AES engine */ 1248 while (src.sg_wa.bytes_left) { 1249 ccp_prepare_data(&src, &dst, &op, unit_size, true); 1250 if (!src.sg_wa.bytes_left) 1251 op.eom = 1; 1252 1253 ret = cmd_q->ccp->vdata->perform->xts_aes(&op); 1254 if (ret) { 1255 cmd->engine_error = cmd_q->cmd_error; 1256 goto e_dst; 1257 } 1258 1259 ccp_process_data(&src, &dst, &op); 1260 } 1261 1262 /* Retrieve the AES context - convert from LE to BE using 1263 * 32-byte (256-bit) byteswapping 1264 */ 1265 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1266 CCP_PASSTHRU_BYTESWAP_256BIT); 1267 if (ret) { 1268 cmd->engine_error = cmd_q->cmd_error; 1269 goto e_dst; 1270 } 1271 1272 /* ...but we only need AES_BLOCK_SIZE bytes */ 1273 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE; 1274 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); 1275 1276e_dst: 1277 if (!in_place) 1278 ccp_free_data(&dst, cmd_q); 1279 1280e_src: 1281 ccp_free_data(&src, cmd_q); 1282 1283e_ctx: 1284 ccp_dm_free(&ctx); 1285 1286e_key: 1287 ccp_dm_free(&key); 1288 1289 return ret; 1290} 1291 1292static noinline_for_stack int 1293ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1294{ 1295 struct ccp_des3_engine *des3 = &cmd->u.des3; 1296 1297 struct ccp_dm_workarea key, ctx; 1298 struct ccp_data src, dst; 1299 struct ccp_op op; 1300 unsigned int dm_offset; 1301 unsigned int len_singlekey; 1302 bool in_place = false; 1303 int ret; 1304 1305 /* Error checks */ 1306 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) 1307 return -EINVAL; 1308 1309 if (!cmd_q->ccp->vdata->perform->des3) 1310 return -EINVAL; 1311 1312 if (des3->key_len != DES3_EDE_KEY_SIZE) 1313 return -EINVAL; 1314 1315 if (((des3->mode == CCP_DES3_MODE_ECB) || 1316 (des3->mode == CCP_DES3_MODE_CBC)) && 1317 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1))) 1318 return -EINVAL; 1319 1320 if (!des3->key || !des3->src || !des3->dst) 1321 return -EINVAL; 1322 1323 if (des3->mode != CCP_DES3_MODE_ECB) { 1324 if (des3->iv_len != DES3_EDE_BLOCK_SIZE) 1325 return -EINVAL; 1326 1327 if (!des3->iv) 1328 return -EINVAL; 1329 } 1330 1331 /* Zero out all the fields of the command desc */ 1332 memset(&op, 0, sizeof(op)); 1333 1334 /* Set up the Function field */ 1335 op.cmd_q = cmd_q; 1336 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1337 op.sb_key = cmd_q->sb_key; 1338 1339 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1; 1340 op.u.des3.type = des3->type; 1341 op.u.des3.mode = des3->mode; 1342 op.u.des3.action = des3->action; 1343 1344 /* 1345 * All supported key sizes fit in a single (32-byte) KSB entry and 1346 * (like AES) must be in little endian format. Use the 256-bit byte 1347 * swap passthru option to convert from big endian to little endian. 1348 */ 1349 ret = ccp_init_dm_workarea(&key, cmd_q, 1350 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES, 1351 DMA_TO_DEVICE); 1352 if (ret) 1353 return ret; 1354 1355 /* 1356 * The contents of the key triplet are in the reverse order of what 1357 * is required by the engine. Copy the 3 pieces individually to put 1358 * them where they belong. 1359 */ 1360 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */ 1361 1362 len_singlekey = des3->key_len / 3; 1363 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey, 1364 des3->key, 0, len_singlekey); 1365 if (ret) 1366 goto e_key; 1367 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey, 1368 des3->key, len_singlekey, len_singlekey); 1369 if (ret) 1370 goto e_key; 1371 ret = ccp_set_dm_area(&key, dm_offset, 1372 des3->key, 2 * len_singlekey, len_singlekey); 1373 if (ret) 1374 goto e_key; 1375 1376 /* Copy the key to the SB */ 1377 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, 1378 CCP_PASSTHRU_BYTESWAP_256BIT); 1379 if (ret) { 1380 cmd->engine_error = cmd_q->cmd_error; 1381 goto e_key; 1382 } 1383 1384 /* 1385 * The DES3 context fits in a single (32-byte) KSB entry and 1386 * must be in little endian format. Use the 256-bit byte swap 1387 * passthru option to convert from big endian to little endian. 1388 */ 1389 if (des3->mode != CCP_DES3_MODE_ECB) { 1390 op.sb_ctx = cmd_q->sb_ctx; 1391 1392 ret = ccp_init_dm_workarea(&ctx, cmd_q, 1393 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES, 1394 DMA_BIDIRECTIONAL); 1395 if (ret) 1396 goto e_key; 1397 1398 /* Load the context into the LSB */ 1399 dm_offset = CCP_SB_BYTES - des3->iv_len; 1400 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, 1401 des3->iv_len); 1402 if (ret) 1403 goto e_ctx; 1404 1405 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1406 CCP_PASSTHRU_BYTESWAP_256BIT); 1407 if (ret) { 1408 cmd->engine_error = cmd_q->cmd_error; 1409 goto e_ctx; 1410 } 1411 } 1412 1413 /* 1414 * Prepare the input and output data workareas. For in-place 1415 * operations we need to set the dma direction to BIDIRECTIONAL 1416 * and copy the src workarea to the dst workarea. 1417 */ 1418 if (sg_virt(des3->src) == sg_virt(des3->dst)) 1419 in_place = true; 1420 1421 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len, 1422 DES3_EDE_BLOCK_SIZE, 1423 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 1424 if (ret) 1425 goto e_ctx; 1426 1427 if (in_place) 1428 dst = src; 1429 else { 1430 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len, 1431 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE); 1432 if (ret) 1433 goto e_src; 1434 } 1435 1436 /* Send data to the CCP DES3 engine */ 1437 while (src.sg_wa.bytes_left) { 1438 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true); 1439 if (!src.sg_wa.bytes_left) { 1440 op.eom = 1; 1441 1442 /* Since we don't retrieve the context in ECB mode 1443 * we have to wait for the operation to complete 1444 * on the last piece of data 1445 */ 1446 op.soc = 0; 1447 } 1448 1449 ret = cmd_q->ccp->vdata->perform->des3(&op); 1450 if (ret) { 1451 cmd->engine_error = cmd_q->cmd_error; 1452 goto e_dst; 1453 } 1454 1455 ccp_process_data(&src, &dst, &op); 1456 } 1457 1458 if (des3->mode != CCP_DES3_MODE_ECB) { 1459 /* Retrieve the context and make BE */ 1460 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1461 CCP_PASSTHRU_BYTESWAP_256BIT); 1462 if (ret) { 1463 cmd->engine_error = cmd_q->cmd_error; 1464 goto e_dst; 1465 } 1466 1467 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ 1468 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, 1469 DES3_EDE_BLOCK_SIZE); 1470 } 1471e_dst: 1472 if (!in_place) 1473 ccp_free_data(&dst, cmd_q); 1474 1475e_src: 1476 ccp_free_data(&src, cmd_q); 1477 1478e_ctx: 1479 if (des3->mode != CCP_DES3_MODE_ECB) 1480 ccp_dm_free(&ctx); 1481 1482e_key: 1483 ccp_dm_free(&key); 1484 1485 return ret; 1486} 1487 1488static noinline_for_stack int 1489ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1490{ 1491 struct ccp_sha_engine *sha = &cmd->u.sha; 1492 struct ccp_dm_workarea ctx; 1493 struct ccp_data src; 1494 struct ccp_op op; 1495 unsigned int ioffset, ooffset; 1496 unsigned int digest_size; 1497 int sb_count; 1498 const void *init; 1499 u64 block_size; 1500 int ctx_size; 1501 int ret; 1502 1503 switch (sha->type) { 1504 case CCP_SHA_TYPE_1: 1505 if (sha->ctx_len < SHA1_DIGEST_SIZE) 1506 return -EINVAL; 1507 block_size = SHA1_BLOCK_SIZE; 1508 break; 1509 case CCP_SHA_TYPE_224: 1510 if (sha->ctx_len < SHA224_DIGEST_SIZE) 1511 return -EINVAL; 1512 block_size = SHA224_BLOCK_SIZE; 1513 break; 1514 case CCP_SHA_TYPE_256: 1515 if (sha->ctx_len < SHA256_DIGEST_SIZE) 1516 return -EINVAL; 1517 block_size = SHA256_BLOCK_SIZE; 1518 break; 1519 case CCP_SHA_TYPE_384: 1520 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1521 || sha->ctx_len < SHA384_DIGEST_SIZE) 1522 return -EINVAL; 1523 block_size = SHA384_BLOCK_SIZE; 1524 break; 1525 case CCP_SHA_TYPE_512: 1526 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0) 1527 || sha->ctx_len < SHA512_DIGEST_SIZE) 1528 return -EINVAL; 1529 block_size = SHA512_BLOCK_SIZE; 1530 break; 1531 default: 1532 return -EINVAL; 1533 } 1534 1535 if (!sha->ctx) 1536 return -EINVAL; 1537 1538 if (!sha->final && (sha->src_len & (block_size - 1))) 1539 return -EINVAL; 1540 1541 /* The version 3 device can't handle zero-length input */ 1542 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { 1543 1544 if (!sha->src_len) { 1545 unsigned int digest_len; 1546 const u8 *sha_zero; 1547 1548 /* Not final, just return */ 1549 if (!sha->final) 1550 return 0; 1551 1552 /* CCP can't do a zero length sha operation so the 1553 * caller must buffer the data. 1554 */ 1555 if (sha->msg_bits) 1556 return -EINVAL; 1557 1558 /* The CCP cannot perform zero-length sha operations 1559 * so the caller is required to buffer data for the 1560 * final operation. However, a sha operation for a 1561 * message with a total length of zero is valid so 1562 * known values are required to supply the result. 1563 */ 1564 switch (sha->type) { 1565 case CCP_SHA_TYPE_1: 1566 sha_zero = sha1_zero_message_hash; 1567 digest_len = SHA1_DIGEST_SIZE; 1568 break; 1569 case CCP_SHA_TYPE_224: 1570 sha_zero = sha224_zero_message_hash; 1571 digest_len = SHA224_DIGEST_SIZE; 1572 break; 1573 case CCP_SHA_TYPE_256: 1574 sha_zero = sha256_zero_message_hash; 1575 digest_len = SHA256_DIGEST_SIZE; 1576 break; 1577 default: 1578 return -EINVAL; 1579 } 1580 1581 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, 1582 digest_len, 1); 1583 1584 return 0; 1585 } 1586 } 1587 1588 /* Set variables used throughout */ 1589 switch (sha->type) { 1590 case CCP_SHA_TYPE_1: 1591 digest_size = SHA1_DIGEST_SIZE; 1592 init = (void *) ccp_sha1_init; 1593 ctx_size = SHA1_DIGEST_SIZE; 1594 sb_count = 1; 1595 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1596 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE; 1597 else 1598 ooffset = ioffset = 0; 1599 break; 1600 case CCP_SHA_TYPE_224: 1601 digest_size = SHA224_DIGEST_SIZE; 1602 init = (void *) ccp_sha224_init; 1603 ctx_size = SHA256_DIGEST_SIZE; 1604 sb_count = 1; 1605 ioffset = 0; 1606 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0)) 1607 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE; 1608 else 1609 ooffset = 0; 1610 break; 1611 case CCP_SHA_TYPE_256: 1612 digest_size = SHA256_DIGEST_SIZE; 1613 init = (void *) ccp_sha256_init; 1614 ctx_size = SHA256_DIGEST_SIZE; 1615 sb_count = 1; 1616 ooffset = ioffset = 0; 1617 break; 1618 case CCP_SHA_TYPE_384: 1619 digest_size = SHA384_DIGEST_SIZE; 1620 init = (void *) ccp_sha384_init; 1621 ctx_size = SHA512_DIGEST_SIZE; 1622 sb_count = 2; 1623 ioffset = 0; 1624 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE; 1625 break; 1626 case CCP_SHA_TYPE_512: 1627 digest_size = SHA512_DIGEST_SIZE; 1628 init = (void *) ccp_sha512_init; 1629 ctx_size = SHA512_DIGEST_SIZE; 1630 sb_count = 2; 1631 ooffset = ioffset = 0; 1632 break; 1633 default: 1634 ret = -EINVAL; 1635 goto e_data; 1636 } 1637 1638 /* For zero-length plaintext the src pointer is ignored; 1639 * otherwise both parts must be valid 1640 */ 1641 if (sha->src_len && !sha->src) 1642 return -EINVAL; 1643 1644 memset(&op, 0, sizeof(op)); 1645 op.cmd_q = cmd_q; 1646 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1647 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */ 1648 op.u.sha.type = sha->type; 1649 op.u.sha.msg_bits = sha->msg_bits; 1650 1651 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry; 1652 * SHA384/512 require 2 adjacent SB slots, with the right half in the 1653 * first slot, and the left half in the second. Each portion must then 1654 * be in little endian format: use the 256-bit byte swap option. 1655 */ 1656 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES, 1657 DMA_BIDIRECTIONAL); 1658 if (ret) 1659 return ret; 1660 if (sha->first) { 1661 switch (sha->type) { 1662 case CCP_SHA_TYPE_1: 1663 case CCP_SHA_TYPE_224: 1664 case CCP_SHA_TYPE_256: 1665 memcpy(ctx.address + ioffset, init, ctx_size); 1666 break; 1667 case CCP_SHA_TYPE_384: 1668 case CCP_SHA_TYPE_512: 1669 memcpy(ctx.address + ctx_size / 2, init, 1670 ctx_size / 2); 1671 memcpy(ctx.address, init + ctx_size / 2, 1672 ctx_size / 2); 1673 break; 1674 default: 1675 ret = -EINVAL; 1676 goto e_ctx; 1677 } 1678 } else { 1679 /* Restore the context */ 1680 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0, 1681 sb_count * CCP_SB_BYTES); 1682 if (ret) 1683 goto e_ctx; 1684 } 1685 1686 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1687 CCP_PASSTHRU_BYTESWAP_256BIT); 1688 if (ret) { 1689 cmd->engine_error = cmd_q->cmd_error; 1690 goto e_ctx; 1691 } 1692 1693 if (sha->src) { 1694 /* Send data to the CCP SHA engine; block_size is set above */ 1695 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, 1696 block_size, DMA_TO_DEVICE); 1697 if (ret) 1698 goto e_ctx; 1699 1700 while (src.sg_wa.bytes_left) { 1701 ccp_prepare_data(&src, NULL, &op, block_size, false); 1702 if (sha->final && !src.sg_wa.bytes_left) 1703 op.eom = 1; 1704 1705 ret = cmd_q->ccp->vdata->perform->sha(&op); 1706 if (ret) { 1707 cmd->engine_error = cmd_q->cmd_error; 1708 goto e_data; 1709 } 1710 1711 ccp_process_data(&src, NULL, &op); 1712 } 1713 } else { 1714 op.eom = 1; 1715 ret = cmd_q->ccp->vdata->perform->sha(&op); 1716 if (ret) { 1717 cmd->engine_error = cmd_q->cmd_error; 1718 goto e_data; 1719 } 1720 } 1721 1722 /* Retrieve the SHA context - convert from LE to BE using 1723 * 32-byte (256-bit) byteswapping to BE 1724 */ 1725 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, 1726 CCP_PASSTHRU_BYTESWAP_256BIT); 1727 if (ret) { 1728 cmd->engine_error = cmd_q->cmd_error; 1729 goto e_data; 1730 } 1731 1732 if (sha->final) { 1733 /* Finishing up, so get the digest */ 1734 switch (sha->type) { 1735 case CCP_SHA_TYPE_1: 1736 case CCP_SHA_TYPE_224: 1737 case CCP_SHA_TYPE_256: 1738 ccp_get_dm_area(&ctx, ooffset, 1739 sha->ctx, 0, 1740 digest_size); 1741 break; 1742 case CCP_SHA_TYPE_384: 1743 case CCP_SHA_TYPE_512: 1744 ccp_get_dm_area(&ctx, 0, 1745 sha->ctx, LSB_ITEM_SIZE - ooffset, 1746 LSB_ITEM_SIZE); 1747 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset, 1748 sha->ctx, 0, 1749 LSB_ITEM_SIZE - ooffset); 1750 break; 1751 default: 1752 ret = -EINVAL; 1753 goto e_data; 1754 } 1755 } else { 1756 /* Stash the context */ 1757 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, 1758 sb_count * CCP_SB_BYTES); 1759 } 1760 1761 if (sha->final && sha->opad) { 1762 /* HMAC operation, recursively perform final SHA */ 1763 struct ccp_cmd hmac_cmd; 1764 struct scatterlist sg; 1765 u8 *hmac_buf; 1766 1767 if (sha->opad_len != block_size) { 1768 ret = -EINVAL; 1769 goto e_data; 1770 } 1771 1772 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); 1773 if (!hmac_buf) { 1774 ret = -ENOMEM; 1775 goto e_data; 1776 } 1777 sg_init_one(&sg, hmac_buf, block_size + digest_size); 1778 1779 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); 1780 switch (sha->type) { 1781 case CCP_SHA_TYPE_1: 1782 case CCP_SHA_TYPE_224: 1783 case CCP_SHA_TYPE_256: 1784 memcpy(hmac_buf + block_size, 1785 ctx.address + ooffset, 1786 digest_size); 1787 break; 1788 case CCP_SHA_TYPE_384: 1789 case CCP_SHA_TYPE_512: 1790 memcpy(hmac_buf + block_size, 1791 ctx.address + LSB_ITEM_SIZE + ooffset, 1792 LSB_ITEM_SIZE); 1793 memcpy(hmac_buf + block_size + 1794 (LSB_ITEM_SIZE - ooffset), 1795 ctx.address, 1796 LSB_ITEM_SIZE); 1797 break; 1798 default: 1799 kfree(hmac_buf); 1800 ret = -EINVAL; 1801 goto e_data; 1802 } 1803 1804 memset(&hmac_cmd, 0, sizeof(hmac_cmd)); 1805 hmac_cmd.engine = CCP_ENGINE_SHA; 1806 hmac_cmd.u.sha.type = sha->type; 1807 hmac_cmd.u.sha.ctx = sha->ctx; 1808 hmac_cmd.u.sha.ctx_len = sha->ctx_len; 1809 hmac_cmd.u.sha.src = &sg; 1810 hmac_cmd.u.sha.src_len = block_size + digest_size; 1811 hmac_cmd.u.sha.opad = NULL; 1812 hmac_cmd.u.sha.opad_len = 0; 1813 hmac_cmd.u.sha.first = 1; 1814 hmac_cmd.u.sha.final = 1; 1815 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; 1816 1817 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); 1818 if (ret) 1819 cmd->engine_error = hmac_cmd.engine_error; 1820 1821 kfree(hmac_buf); 1822 } 1823 1824e_data: 1825 if (sha->src) 1826 ccp_free_data(&src, cmd_q); 1827 1828e_ctx: 1829 ccp_dm_free(&ctx); 1830 1831 return ret; 1832} 1833 1834static noinline_for_stack int 1835ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1836{ 1837 struct ccp_rsa_engine *rsa = &cmd->u.rsa; 1838 struct ccp_dm_workarea exp, src, dst; 1839 struct ccp_op op; 1840 unsigned int sb_count, i_len, o_len; 1841 int ret; 1842 1843 /* Check against the maximum allowable size, in bits */ 1844 if (rsa->key_size > cmd_q->ccp->vdata->rsamax) 1845 return -EINVAL; 1846 1847 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) 1848 return -EINVAL; 1849 1850 memset(&op, 0, sizeof(op)); 1851 op.cmd_q = cmd_q; 1852 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1853 1854 /* The RSA modulus must precede the message being acted upon, so 1855 * it must be copied to a DMA area where the message and the 1856 * modulus can be concatenated. Therefore the input buffer 1857 * length required is twice the output buffer length (which 1858 * must be a multiple of 256-bits). Compute o_len, i_len in bytes. 1859 * Buffer sizes must be a multiple of 32 bytes; rounding up may be 1860 * required. 1861 */ 1862 o_len = 32 * ((rsa->key_size + 255) / 256); 1863 i_len = o_len * 2; 1864 1865 sb_count = 0; 1866 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1867 /* sb_count is the number of storage block slots required 1868 * for the modulus. 1869 */ 1870 sb_count = o_len / CCP_SB_BYTES; 1871 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, 1872 sb_count); 1873 if (!op.sb_key) 1874 return -EIO; 1875 } else { 1876 /* A version 5 device allows a modulus size that will not fit 1877 * in the LSB, so the command will transfer it from memory. 1878 * Set the sb key to the default, even though it's not used. 1879 */ 1880 op.sb_key = cmd_q->sb_key; 1881 } 1882 1883 /* The RSA exponent must be in little endian format. Reverse its 1884 * byte order. 1885 */ 1886 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); 1887 if (ret) 1888 goto e_sb; 1889 1890 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); 1891 if (ret) 1892 goto e_exp; 1893 1894 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { 1895 /* Copy the exponent to the local storage block, using 1896 * as many 32-byte blocks as were allocated above. It's 1897 * already little endian, so no further change is required. 1898 */ 1899 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, 1900 CCP_PASSTHRU_BYTESWAP_NOOP); 1901 if (ret) { 1902 cmd->engine_error = cmd_q->cmd_error; 1903 goto e_exp; 1904 } 1905 } else { 1906 /* The exponent can be retrieved from memory via DMA. */ 1907 op.exp.u.dma.address = exp.dma.address; 1908 op.exp.u.dma.offset = 0; 1909 } 1910 1911 /* Concatenate the modulus and the message. Both the modulus and 1912 * the operands must be in little endian format. Since the input 1913 * is in big endian format it must be converted. 1914 */ 1915 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); 1916 if (ret) 1917 goto e_exp; 1918 1919 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len); 1920 if (ret) 1921 goto e_src; 1922 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len); 1923 if (ret) 1924 goto e_src; 1925 1926 /* Prepare the output area for the operation */ 1927 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); 1928 if (ret) 1929 goto e_src; 1930 1931 op.soc = 1; 1932 op.src.u.dma.address = src.dma.address; 1933 op.src.u.dma.offset = 0; 1934 op.src.u.dma.length = i_len; 1935 op.dst.u.dma.address = dst.dma.address; 1936 op.dst.u.dma.offset = 0; 1937 op.dst.u.dma.length = o_len; 1938 1939 op.u.rsa.mod_size = rsa->key_size; 1940 op.u.rsa.input_len = i_len; 1941 1942 ret = cmd_q->ccp->vdata->perform->rsa(&op); 1943 if (ret) { 1944 cmd->engine_error = cmd_q->cmd_error; 1945 goto e_dst; 1946 } 1947 1948 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); 1949 1950e_dst: 1951 ccp_dm_free(&dst); 1952 1953e_src: 1954 ccp_dm_free(&src); 1955 1956e_exp: 1957 ccp_dm_free(&exp); 1958 1959e_sb: 1960 if (sb_count) 1961 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); 1962 1963 return ret; 1964} 1965 1966static noinline_for_stack int 1967ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 1968{ 1969 struct ccp_passthru_engine *pt = &cmd->u.passthru; 1970 struct ccp_dm_workarea mask; 1971 struct ccp_data src, dst; 1972 struct ccp_op op; 1973 bool in_place = false; 1974 unsigned int i; 1975 int ret = 0; 1976 1977 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 1978 return -EINVAL; 1979 1980 if (!pt->src || !pt->dst) 1981 return -EINVAL; 1982 1983 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1984 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 1985 return -EINVAL; 1986 if (!pt->mask) 1987 return -EINVAL; 1988 } 1989 1990 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 1991 1992 memset(&op, 0, sizeof(op)); 1993 op.cmd_q = cmd_q; 1994 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 1995 1996 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 1997 /* Load the mask */ 1998 op.sb_key = cmd_q->sb_key; 1999 2000 ret = ccp_init_dm_workarea(&mask, cmd_q, 2001 CCP_PASSTHRU_SB_COUNT * 2002 CCP_SB_BYTES, 2003 DMA_TO_DEVICE); 2004 if (ret) 2005 return ret; 2006 2007 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); 2008 if (ret) 2009 goto e_mask; 2010 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2011 CCP_PASSTHRU_BYTESWAP_NOOP); 2012 if (ret) { 2013 cmd->engine_error = cmd_q->cmd_error; 2014 goto e_mask; 2015 } 2016 } 2017 2018 /* Prepare the input and output data workareas. For in-place 2019 * operations we need to set the dma direction to BIDIRECTIONAL 2020 * and copy the src workarea to the dst workarea. 2021 */ 2022 if (sg_virt(pt->src) == sg_virt(pt->dst)) 2023 in_place = true; 2024 2025 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, 2026 CCP_PASSTHRU_MASKSIZE, 2027 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); 2028 if (ret) 2029 goto e_mask; 2030 2031 if (in_place) { 2032 dst = src; 2033 } else { 2034 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 2035 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 2036 if (ret) 2037 goto e_src; 2038 } 2039 2040 /* Send data to the CCP Passthru engine 2041 * Because the CCP engine works on a single source and destination 2042 * dma address at a time, each entry in the source scatterlist 2043 * (after the dma_map_sg call) must be less than or equal to the 2044 * (remaining) length in the destination scatterlist entry and the 2045 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE 2046 */ 2047 dst.sg_wa.sg_used = 0; 2048 for (i = 1; i <= src.sg_wa.dma_count; i++) { 2049 if (!dst.sg_wa.sg || 2050 (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) { 2051 ret = -EINVAL; 2052 goto e_dst; 2053 } 2054 2055 if (i == src.sg_wa.dma_count) { 2056 op.eom = 1; 2057 op.soc = 1; 2058 } 2059 2060 op.src.type = CCP_MEMTYPE_SYSTEM; 2061 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); 2062 op.src.u.dma.offset = 0; 2063 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); 2064 2065 op.dst.type = CCP_MEMTYPE_SYSTEM; 2066 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); 2067 op.dst.u.dma.offset = dst.sg_wa.sg_used; 2068 op.dst.u.dma.length = op.src.u.dma.length; 2069 2070 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2071 if (ret) { 2072 cmd->engine_error = cmd_q->cmd_error; 2073 goto e_dst; 2074 } 2075 2076 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg); 2077 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) { 2078 dst.sg_wa.sg = sg_next(dst.sg_wa.sg); 2079 dst.sg_wa.sg_used = 0; 2080 } 2081 src.sg_wa.sg = sg_next(src.sg_wa.sg); 2082 } 2083 2084e_dst: 2085 if (!in_place) 2086 ccp_free_data(&dst, cmd_q); 2087 2088e_src: 2089 ccp_free_data(&src, cmd_q); 2090 2091e_mask: 2092 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 2093 ccp_dm_free(&mask); 2094 2095 return ret; 2096} 2097 2098static noinline_for_stack int 2099ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, 2100 struct ccp_cmd *cmd) 2101{ 2102 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap; 2103 struct ccp_dm_workarea mask; 2104 struct ccp_op op; 2105 int ret; 2106 2107 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) 2108 return -EINVAL; 2109 2110 if (!pt->src_dma || !pt->dst_dma) 2111 return -EINVAL; 2112 2113 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2114 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) 2115 return -EINVAL; 2116 if (!pt->mask) 2117 return -EINVAL; 2118 } 2119 2120 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1); 2121 2122 memset(&op, 0, sizeof(op)); 2123 op.cmd_q = cmd_q; 2124 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2125 2126 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { 2127 /* Load the mask */ 2128 op.sb_key = cmd_q->sb_key; 2129 2130 mask.length = pt->mask_len; 2131 mask.dma.address = pt->mask; 2132 mask.dma.length = pt->mask_len; 2133 2134 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key, 2135 CCP_PASSTHRU_BYTESWAP_NOOP); 2136 if (ret) { 2137 cmd->engine_error = cmd_q->cmd_error; 2138 return ret; 2139 } 2140 } 2141 2142 /* Send data to the CCP Passthru engine */ 2143 op.eom = 1; 2144 op.soc = 1; 2145 2146 op.src.type = CCP_MEMTYPE_SYSTEM; 2147 op.src.u.dma.address = pt->src_dma; 2148 op.src.u.dma.offset = 0; 2149 op.src.u.dma.length = pt->src_len; 2150 2151 op.dst.type = CCP_MEMTYPE_SYSTEM; 2152 op.dst.u.dma.address = pt->dst_dma; 2153 op.dst.u.dma.offset = 0; 2154 op.dst.u.dma.length = pt->src_len; 2155 2156 ret = cmd_q->ccp->vdata->perform->passthru(&op); 2157 if (ret) 2158 cmd->engine_error = cmd_q->cmd_error; 2159 2160 return ret; 2161} 2162 2163static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2164{ 2165 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2166 struct ccp_dm_workarea src, dst; 2167 struct ccp_op op; 2168 int ret; 2169 u8 *save; 2170 2171 if (!ecc->u.mm.operand_1 || 2172 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) 2173 return -EINVAL; 2174 2175 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) 2176 if (!ecc->u.mm.operand_2 || 2177 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) 2178 return -EINVAL; 2179 2180 if (!ecc->u.mm.result || 2181 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) 2182 return -EINVAL; 2183 2184 memset(&op, 0, sizeof(op)); 2185 op.cmd_q = cmd_q; 2186 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2187 2188 /* Concatenate the modulus and the operands. Both the modulus and 2189 * the operands must be in little endian format. Since the input 2190 * is in big endian format it must be converted and placed in a 2191 * fixed length buffer. 2192 */ 2193 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2194 DMA_TO_DEVICE); 2195 if (ret) 2196 return ret; 2197 2198 /* Save the workarea address since it is updated in order to perform 2199 * the concatenation 2200 */ 2201 save = src.address; 2202 2203 /* Copy the ECC modulus */ 2204 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2205 if (ret) 2206 goto e_src; 2207 src.address += CCP_ECC_OPERAND_SIZE; 2208 2209 /* Copy the first operand */ 2210 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0, 2211 ecc->u.mm.operand_1_len); 2212 if (ret) 2213 goto e_src; 2214 src.address += CCP_ECC_OPERAND_SIZE; 2215 2216 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { 2217 /* Copy the second operand */ 2218 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0, 2219 ecc->u.mm.operand_2_len); 2220 if (ret) 2221 goto e_src; 2222 src.address += CCP_ECC_OPERAND_SIZE; 2223 } 2224 2225 /* Restore the workarea address */ 2226 src.address = save; 2227 2228 /* Prepare the output area for the operation */ 2229 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2230 DMA_FROM_DEVICE); 2231 if (ret) 2232 goto e_src; 2233 2234 op.soc = 1; 2235 op.src.u.dma.address = src.dma.address; 2236 op.src.u.dma.offset = 0; 2237 op.src.u.dma.length = src.length; 2238 op.dst.u.dma.address = dst.dma.address; 2239 op.dst.u.dma.offset = 0; 2240 op.dst.u.dma.length = dst.length; 2241 2242 op.u.ecc.function = cmd->u.ecc.function; 2243 2244 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2245 if (ret) { 2246 cmd->engine_error = cmd_q->cmd_error; 2247 goto e_dst; 2248 } 2249 2250 ecc->ecc_result = le16_to_cpup( 2251 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2252 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2253 ret = -EIO; 2254 goto e_dst; 2255 } 2256 2257 /* Save the ECC result */ 2258 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0, 2259 CCP_ECC_MODULUS_BYTES); 2260 2261e_dst: 2262 ccp_dm_free(&dst); 2263 2264e_src: 2265 ccp_dm_free(&src); 2266 2267 return ret; 2268} 2269 2270static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2271{ 2272 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2273 struct ccp_dm_workarea src, dst; 2274 struct ccp_op op; 2275 int ret; 2276 u8 *save; 2277 2278 if (!ecc->u.pm.point_1.x || 2279 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || 2280 !ecc->u.pm.point_1.y || 2281 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) 2282 return -EINVAL; 2283 2284 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2285 if (!ecc->u.pm.point_2.x || 2286 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || 2287 !ecc->u.pm.point_2.y || 2288 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) 2289 return -EINVAL; 2290 } else { 2291 if (!ecc->u.pm.domain_a || 2292 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) 2293 return -EINVAL; 2294 2295 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) 2296 if (!ecc->u.pm.scalar || 2297 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) 2298 return -EINVAL; 2299 } 2300 2301 if (!ecc->u.pm.result.x || 2302 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || 2303 !ecc->u.pm.result.y || 2304 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) 2305 return -EINVAL; 2306 2307 memset(&op, 0, sizeof(op)); 2308 op.cmd_q = cmd_q; 2309 op.jobid = CCP_NEW_JOBID(cmd_q->ccp); 2310 2311 /* Concatenate the modulus and the operands. Both the modulus and 2312 * the operands must be in little endian format. Since the input 2313 * is in big endian format it must be converted and placed in a 2314 * fixed length buffer. 2315 */ 2316 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, 2317 DMA_TO_DEVICE); 2318 if (ret) 2319 return ret; 2320 2321 /* Save the workarea address since it is updated in order to perform 2322 * the concatenation 2323 */ 2324 save = src.address; 2325 2326 /* Copy the ECC modulus */ 2327 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len); 2328 if (ret) 2329 goto e_src; 2330 src.address += CCP_ECC_OPERAND_SIZE; 2331 2332 /* Copy the first point X and Y coordinate */ 2333 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0, 2334 ecc->u.pm.point_1.x_len); 2335 if (ret) 2336 goto e_src; 2337 src.address += CCP_ECC_OPERAND_SIZE; 2338 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0, 2339 ecc->u.pm.point_1.y_len); 2340 if (ret) 2341 goto e_src; 2342 src.address += CCP_ECC_OPERAND_SIZE; 2343 2344 /* Set the first point Z coordinate to 1 */ 2345 *src.address = 0x01; 2346 src.address += CCP_ECC_OPERAND_SIZE; 2347 2348 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 2349 /* Copy the second point X and Y coordinate */ 2350 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0, 2351 ecc->u.pm.point_2.x_len); 2352 if (ret) 2353 goto e_src; 2354 src.address += CCP_ECC_OPERAND_SIZE; 2355 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0, 2356 ecc->u.pm.point_2.y_len); 2357 if (ret) 2358 goto e_src; 2359 src.address += CCP_ECC_OPERAND_SIZE; 2360 2361 /* Set the second point Z coordinate to 1 */ 2362 *src.address = 0x01; 2363 src.address += CCP_ECC_OPERAND_SIZE; 2364 } else { 2365 /* Copy the Domain "a" parameter */ 2366 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0, 2367 ecc->u.pm.domain_a_len); 2368 if (ret) 2369 goto e_src; 2370 src.address += CCP_ECC_OPERAND_SIZE; 2371 2372 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { 2373 /* Copy the scalar value */ 2374 ret = ccp_reverse_set_dm_area(&src, 0, 2375 ecc->u.pm.scalar, 0, 2376 ecc->u.pm.scalar_len); 2377 if (ret) 2378 goto e_src; 2379 src.address += CCP_ECC_OPERAND_SIZE; 2380 } 2381 } 2382 2383 /* Restore the workarea address */ 2384 src.address = save; 2385 2386 /* Prepare the output area for the operation */ 2387 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, 2388 DMA_FROM_DEVICE); 2389 if (ret) 2390 goto e_src; 2391 2392 op.soc = 1; 2393 op.src.u.dma.address = src.dma.address; 2394 op.src.u.dma.offset = 0; 2395 op.src.u.dma.length = src.length; 2396 op.dst.u.dma.address = dst.dma.address; 2397 op.dst.u.dma.offset = 0; 2398 op.dst.u.dma.length = dst.length; 2399 2400 op.u.ecc.function = cmd->u.ecc.function; 2401 2402 ret = cmd_q->ccp->vdata->perform->ecc(&op); 2403 if (ret) { 2404 cmd->engine_error = cmd_q->cmd_error; 2405 goto e_dst; 2406 } 2407 2408 ecc->ecc_result = le16_to_cpup( 2409 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); 2410 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { 2411 ret = -EIO; 2412 goto e_dst; 2413 } 2414 2415 /* Save the workarea address since it is updated as we walk through 2416 * to copy the point math result 2417 */ 2418 save = dst.address; 2419 2420 /* Save the ECC result X and Y coordinates */ 2421 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0, 2422 CCP_ECC_MODULUS_BYTES); 2423 dst.address += CCP_ECC_OUTPUT_SIZE; 2424 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0, 2425 CCP_ECC_MODULUS_BYTES); 2426 2427 /* Restore the workarea address */ 2428 dst.address = save; 2429 2430e_dst: 2431 ccp_dm_free(&dst); 2432 2433e_src: 2434 ccp_dm_free(&src); 2435 2436 return ret; 2437} 2438 2439static noinline_for_stack int 2440ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2441{ 2442 struct ccp_ecc_engine *ecc = &cmd->u.ecc; 2443 2444 ecc->ecc_result = 0; 2445 2446 if (!ecc->mod || 2447 (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) 2448 return -EINVAL; 2449 2450 switch (ecc->function) { 2451 case CCP_ECC_FUNCTION_MMUL_384BIT: 2452 case CCP_ECC_FUNCTION_MADD_384BIT: 2453 case CCP_ECC_FUNCTION_MINV_384BIT: 2454 return ccp_run_ecc_mm_cmd(cmd_q, cmd); 2455 2456 case CCP_ECC_FUNCTION_PADD_384BIT: 2457 case CCP_ECC_FUNCTION_PMUL_384BIT: 2458 case CCP_ECC_FUNCTION_PDBL_384BIT: 2459 return ccp_run_ecc_pm_cmd(cmd_q, cmd); 2460 2461 default: 2462 return -EINVAL; 2463 } 2464} 2465 2466int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) 2467{ 2468 int ret; 2469 2470 cmd->engine_error = 0; 2471 cmd_q->cmd_error = 0; 2472 cmd_q->int_rcvd = 0; 2473 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q); 2474 2475 switch (cmd->engine) { 2476 case CCP_ENGINE_AES: 2477 switch (cmd->u.aes.mode) { 2478 case CCP_AES_MODE_CMAC: 2479 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd); 2480 break; 2481 case CCP_AES_MODE_GCM: 2482 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd); 2483 break; 2484 default: 2485 ret = ccp_run_aes_cmd(cmd_q, cmd); 2486 break; 2487 } 2488 break; 2489 case CCP_ENGINE_XTS_AES_128: 2490 ret = ccp_run_xts_aes_cmd(cmd_q, cmd); 2491 break; 2492 case CCP_ENGINE_DES3: 2493 ret = ccp_run_des3_cmd(cmd_q, cmd); 2494 break; 2495 case CCP_ENGINE_SHA: 2496 ret = ccp_run_sha_cmd(cmd_q, cmd); 2497 break; 2498 case CCP_ENGINE_RSA: 2499 ret = ccp_run_rsa_cmd(cmd_q, cmd); 2500 break; 2501 case CCP_ENGINE_PASSTHRU: 2502 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP) 2503 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd); 2504 else 2505 ret = ccp_run_passthru_cmd(cmd_q, cmd); 2506 break; 2507 case CCP_ENGINE_ECC: 2508 ret = ccp_run_ecc_cmd(cmd_q, cmd); 2509 break; 2510 default: 2511 ret = -EINVAL; 2512 } 2513 2514 return ret; 2515} 2516