1// SPDX-License-Identifier: GPL-2.0 2/* 3 * sun8i-ss-hash.c - hardware cryptographic offloader for 4 * Allwinner A80/A83T SoC 5 * 6 * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com> 7 * 8 * This file add support for MD5 and SHA1/SHA224/SHA256. 9 * 10 * You could find the datasheet in Documentation/arch/arm/sunxi.rst 11 */ 12 13#include <crypto/hmac.h> 14#include <crypto/internal/hash.h> 15#include <crypto/md5.h> 16#include <crypto/scatterwalk.h> 17#include <crypto/sha1.h> 18#include <crypto/sha2.h> 19#include <linux/bottom_half.h> 20#include <linux/dma-mapping.h> 21#include <linux/err.h> 22#include <linux/kernel.h> 23#include <linux/pm_runtime.h> 24#include <linux/scatterlist.h> 25#include <linux/slab.h> 26#include <linux/string.h> 27#include "sun8i-ss.h" 28 29static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key, 30 unsigned int keylen) 31{ 32 struct crypto_shash *xtfm; 33 int ret; 34 35 xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK); 36 if (IS_ERR(xtfm)) 37 return PTR_ERR(xtfm); 38 39 ret = crypto_shash_tfm_digest(xtfm, key, keylen, tfmctx->key); 40 if (ret) 41 dev_err(tfmctx->ss->dev, "shash digest error ret=%d\n", ret); 42 43 crypto_free_shash(xtfm); 44 return ret; 45} 46 47int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, 48 unsigned int keylen) 49{ 50 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(ahash); 51 int digestsize, i; 52 int bs = crypto_ahash_blocksize(ahash); 53 int ret; 54 55 digestsize = crypto_ahash_digestsize(ahash); 56 57 if (keylen > bs) { 58 ret = sun8i_ss_hashkey(tfmctx, key, keylen); 59 if (ret) 60 return ret; 61 tfmctx->keylen = digestsize; 62 } else { 63 tfmctx->keylen = keylen; 64 memcpy(tfmctx->key, key, keylen); 65 } 66 67 tfmctx->ipad = kzalloc(bs, GFP_KERNEL); 68 if (!tfmctx->ipad) 69 return -ENOMEM; 70 tfmctx->opad = kzalloc(bs, GFP_KERNEL); 71 if (!tfmctx->opad) { 72 ret = -ENOMEM; 73 goto err_opad; 74 } 75 76 memset(tfmctx->key + tfmctx->keylen, 0, bs - tfmctx->keylen); 77 memcpy(tfmctx->ipad, tfmctx->key, tfmctx->keylen); 78 memcpy(tfmctx->opad, tfmctx->key, tfmctx->keylen); 79 for (i = 0; i < bs; i++) { 80 tfmctx->ipad[i] ^= HMAC_IPAD_VALUE; 81 tfmctx->opad[i] ^= HMAC_OPAD_VALUE; 82 } 83 84 ret = crypto_ahash_setkey(tfmctx->fallback_tfm, key, keylen); 85 if (!ret) 86 return 0; 87 88 memzero_explicit(tfmctx->key, keylen); 89 kfree_sensitive(tfmctx->opad); 90err_opad: 91 kfree_sensitive(tfmctx->ipad); 92 return ret; 93} 94 95int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm) 96{ 97 struct sun8i_ss_hash_tfm_ctx *op = crypto_ahash_ctx(tfm); 98 struct ahash_alg *alg = crypto_ahash_alg(tfm); 99 struct sun8i_ss_alg_template *algt; 100 int err; 101 102 algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); 103 op->ss = algt->ss; 104 105 /* FALLBACK */ 106 op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0, 107 CRYPTO_ALG_NEED_FALLBACK); 108 if (IS_ERR(op->fallback_tfm)) { 109 dev_err(algt->ss->dev, "Fallback driver could no be loaded\n"); 110 return PTR_ERR(op->fallback_tfm); 111 } 112 113 crypto_ahash_set_statesize(tfm, 114 crypto_ahash_statesize(op->fallback_tfm)); 115 116 crypto_ahash_set_reqsize(tfm, 117 sizeof(struct sun8i_ss_hash_reqctx) + 118 crypto_ahash_reqsize(op->fallback_tfm)); 119 120 memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm), 121 CRYPTO_MAX_ALG_NAME); 122 123 err = pm_runtime_get_sync(op->ss->dev); 124 if (err < 0) 125 goto error_pm; 126 return 0; 127error_pm: 128 pm_runtime_put_noidle(op->ss->dev); 129 crypto_free_ahash(op->fallback_tfm); 130 return err; 131} 132 133void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm) 134{ 135 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 136 137 kfree_sensitive(tfmctx->ipad); 138 kfree_sensitive(tfmctx->opad); 139 140 crypto_free_ahash(tfmctx->fallback_tfm); 141 pm_runtime_put_sync_suspend(tfmctx->ss->dev); 142} 143 144int sun8i_ss_hash_init(struct ahash_request *areq) 145{ 146 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 147 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 148 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 149 150 memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx)); 151 152 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 153 rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 154 155 return crypto_ahash_init(&rctx->fallback_req); 156} 157 158int sun8i_ss_hash_export(struct ahash_request *areq, void *out) 159{ 160 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 161 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 162 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 163 164 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 165 rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 166 167 return crypto_ahash_export(&rctx->fallback_req, out); 168} 169 170int sun8i_ss_hash_import(struct ahash_request *areq, const void *in) 171{ 172 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 173 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 174 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 175 176 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 177 rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 178 179 return crypto_ahash_import(&rctx->fallback_req, in); 180} 181 182int sun8i_ss_hash_final(struct ahash_request *areq) 183{ 184 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 185 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 186 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 187 188 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 189 rctx->fallback_req.base.flags = areq->base.flags & 190 CRYPTO_TFM_REQ_MAY_SLEEP; 191 rctx->fallback_req.result = areq->result; 192 193 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { 194 struct ahash_alg *alg = crypto_ahash_alg(tfm); 195 struct sun8i_ss_alg_template *algt __maybe_unused; 196 197 algt = container_of(alg, struct sun8i_ss_alg_template, 198 alg.hash.base); 199 200#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG 201 algt->stat_fb++; 202#endif 203 } 204 205 return crypto_ahash_final(&rctx->fallback_req); 206} 207 208int sun8i_ss_hash_update(struct ahash_request *areq) 209{ 210 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 211 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 212 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 213 214 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 215 rctx->fallback_req.base.flags = areq->base.flags & 216 CRYPTO_TFM_REQ_MAY_SLEEP; 217 rctx->fallback_req.nbytes = areq->nbytes; 218 rctx->fallback_req.src = areq->src; 219 220 return crypto_ahash_update(&rctx->fallback_req); 221} 222 223int sun8i_ss_hash_finup(struct ahash_request *areq) 224{ 225 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 226 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 227 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 228 229 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 230 rctx->fallback_req.base.flags = areq->base.flags & 231 CRYPTO_TFM_REQ_MAY_SLEEP; 232 233 rctx->fallback_req.nbytes = areq->nbytes; 234 rctx->fallback_req.src = areq->src; 235 rctx->fallback_req.result = areq->result; 236 237 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { 238 struct ahash_alg *alg = crypto_ahash_alg(tfm); 239 struct sun8i_ss_alg_template *algt __maybe_unused; 240 241 algt = container_of(alg, struct sun8i_ss_alg_template, 242 alg.hash.base); 243 244#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG 245 algt->stat_fb++; 246#endif 247 } 248 249 return crypto_ahash_finup(&rctx->fallback_req); 250} 251 252static int sun8i_ss_hash_digest_fb(struct ahash_request *areq) 253{ 254 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 255 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 256 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 257 258 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); 259 rctx->fallback_req.base.flags = areq->base.flags & 260 CRYPTO_TFM_REQ_MAY_SLEEP; 261 262 rctx->fallback_req.nbytes = areq->nbytes; 263 rctx->fallback_req.src = areq->src; 264 rctx->fallback_req.result = areq->result; 265 266 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) { 267 struct ahash_alg *alg = crypto_ahash_alg(tfm); 268 struct sun8i_ss_alg_template *algt __maybe_unused; 269 270 algt = container_of(alg, struct sun8i_ss_alg_template, 271 alg.hash.base); 272 273#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG 274 algt->stat_fb++; 275#endif 276 } 277 278 return crypto_ahash_digest(&rctx->fallback_req); 279} 280 281static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss, 282 struct sun8i_ss_hash_reqctx *rctx, 283 const char *name) 284{ 285 int flow = rctx->flow; 286 u32 v = SS_START; 287 int i; 288 289#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG 290 ss->flows[flow].stat_req++; 291#endif 292 293 /* choose between stream0/stream1 */ 294 if (flow) 295 v |= SS_FLOW1; 296 else 297 v |= SS_FLOW0; 298 299 v |= rctx->method; 300 301 for (i = 0; i < MAX_SG; i++) { 302 if (!rctx->t_dst[i].addr) 303 break; 304 305 mutex_lock(&ss->mlock); 306 if (i > 0) { 307 v |= BIT(17); 308 writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG); 309 writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG); 310 } 311 312 dev_dbg(ss->dev, 313 "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n", 314 i, flow, name, v, 315 rctx->t_src[i].len, rctx->t_dst[i].len, 316 rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr); 317 318 writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG); 319 writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG); 320 writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG); 321 writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG); 322 323 reinit_completion(&ss->flows[flow].complete); 324 ss->flows[flow].status = 0; 325 wmb(); 326 327 writel(v, ss->base + SS_CTL_REG); 328 mutex_unlock(&ss->mlock); 329 wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, 330 msecs_to_jiffies(2000)); 331 if (ss->flows[flow].status == 0) { 332 dev_err(ss->dev, "DMA timeout for %s\n", name); 333 return -EFAULT; 334 } 335 } 336 337 return 0; 338} 339 340static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq) 341{ 342 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 343 struct ahash_alg *alg = crypto_ahash_alg(tfm); 344 struct sun8i_ss_alg_template *algt; 345 struct scatterlist *sg; 346 347 algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); 348 349 if (areq->nbytes == 0) { 350 algt->stat_fb_len++; 351 return true; 352 } 353 354 if (areq->nbytes >= MAX_PAD_SIZE - 64) { 355 algt->stat_fb_len++; 356 return true; 357 } 358 359 /* we need to reserve one SG for the padding one */ 360 if (sg_nents(areq->src) > MAX_SG - 1) { 361 algt->stat_fb_sgnum++; 362 return true; 363 } 364 365 sg = areq->src; 366 while (sg) { 367 /* SS can operate hash only on full block size 368 * since SS support only MD5,sha1,sha224 and sha256, blocksize 369 * is always 64 370 */ 371 /* Only the last block could be bounced to the pad buffer */ 372 if (sg->length % 64 && sg_next(sg)) { 373 algt->stat_fb_sglen++; 374 return true; 375 } 376 if (!IS_ALIGNED(sg->offset, sizeof(u32))) { 377 algt->stat_fb_align++; 378 return true; 379 } 380 if (sg->length % 4) { 381 algt->stat_fb_sglen++; 382 return true; 383 } 384 sg = sg_next(sg); 385 } 386 return false; 387} 388 389int sun8i_ss_hash_digest(struct ahash_request *areq) 390{ 391 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 392 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 393 struct ahash_alg *alg = crypto_ahash_alg(tfm); 394 struct sun8i_ss_alg_template *algt; 395 struct sun8i_ss_dev *ss; 396 struct crypto_engine *engine; 397 int e; 398 399 if (sun8i_ss_hash_need_fallback(areq)) 400 return sun8i_ss_hash_digest_fb(areq); 401 402 algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); 403 ss = algt->ss; 404 405 e = sun8i_ss_get_engine_number(ss); 406 rctx->flow = e; 407 engine = ss->flows[e].engine; 408 409 return crypto_transfer_hash_request_to_engine(engine, areq); 410} 411 412static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs) 413{ 414 u64 fill, min_fill, j, k; 415 __be64 *bebits; 416 __le64 *lebits; 417 418 j = padi; 419 buf[j++] = cpu_to_le32(0x80); 420 421 if (bs == 64) { 422 fill = 64 - (byte_count % 64); 423 min_fill = 2 * sizeof(u32) + sizeof(u32); 424 } else { 425 fill = 128 - (byte_count % 128); 426 min_fill = 4 * sizeof(u32) + sizeof(u32); 427 } 428 429 if (fill < min_fill) 430 fill += bs; 431 432 k = j; 433 j += (fill - min_fill) / sizeof(u32); 434 if (j * 4 > bufsize) { 435 pr_err("%s OVERFLOW %llu\n", __func__, j); 436 return 0; 437 } 438 for (; k < j; k++) 439 buf[k] = 0; 440 441 if (le) { 442 /* MD5 */ 443 lebits = (__le64 *)&buf[j]; 444 *lebits = cpu_to_le64(byte_count << 3); 445 j += 2; 446 } else { 447 if (bs == 64) { 448 /* sha1 sha224 sha256 */ 449 bebits = (__be64 *)&buf[j]; 450 *bebits = cpu_to_be64(byte_count << 3); 451 j += 2; 452 } else { 453 /* sha384 sha512*/ 454 bebits = (__be64 *)&buf[j]; 455 *bebits = cpu_to_be64(byte_count >> 61); 456 j += 2; 457 bebits = (__be64 *)&buf[j]; 458 *bebits = cpu_to_be64(byte_count << 3); 459 j += 2; 460 } 461 } 462 if (j * 4 > bufsize) { 463 pr_err("%s OVERFLOW %llu\n", __func__, j); 464 return 0; 465 } 466 467 return j; 468} 469 470/* sun8i_ss_hash_run - run an ahash request 471 * Send the data of the request to the SS along with an extra SG with padding 472 */ 473int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) 474{ 475 struct ahash_request *areq = container_of(breq, struct ahash_request, base); 476 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 477 struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); 478 struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq); 479 struct ahash_alg *alg = crypto_ahash_alg(tfm); 480 struct sun8i_ss_alg_template *algt; 481 struct sun8i_ss_dev *ss; 482 struct scatterlist *sg; 483 int bs = crypto_ahash_blocksize(tfm); 484 int nr_sgs, err, digestsize; 485 unsigned int len; 486 u64 byte_count; 487 void *pad, *result; 488 int j, i, k, todo; 489 dma_addr_t addr_res, addr_pad, addr_xpad; 490 __le32 *bf; 491 /* HMAC step: 492 * 0: normal hashing 493 * 1: IPAD 494 * 2: OPAD 495 */ 496 int hmac = 0; 497 498 algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); 499 ss = algt->ss; 500 501 digestsize = crypto_ahash_digestsize(tfm); 502 if (digestsize == SHA224_DIGEST_SIZE) 503 digestsize = SHA256_DIGEST_SIZE; 504 505 result = ss->flows[rctx->flow].result; 506 pad = ss->flows[rctx->flow].pad; 507 bf = (__le32 *)pad; 508 509 for (i = 0; i < MAX_SG; i++) { 510 rctx->t_dst[i].addr = 0; 511 rctx->t_dst[i].len = 0; 512 } 513 514#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG 515 algt->stat_req++; 516#endif 517 518 rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; 519 520 nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); 521 if (nr_sgs <= 0 || nr_sgs > MAX_SG) { 522 dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); 523 err = -EINVAL; 524 goto theend; 525 } 526 527 addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); 528 if (dma_mapping_error(ss->dev, addr_res)) { 529 dev_err(ss->dev, "DMA map dest\n"); 530 err = -EINVAL; 531 goto err_dma_result; 532 } 533 534 j = 0; 535 len = areq->nbytes; 536 sg = areq->src; 537 i = 0; 538 while (len > 0 && sg) { 539 if (sg_dma_len(sg) == 0) { 540 sg = sg_next(sg); 541 continue; 542 } 543 todo = min(len, sg_dma_len(sg)); 544 /* only the last SG could be with a size not modulo64 */ 545 if (todo % 64 == 0) { 546 rctx->t_src[i].addr = sg_dma_address(sg); 547 rctx->t_src[i].len = todo / 4; 548 rctx->t_dst[i].addr = addr_res; 549 rctx->t_dst[i].len = digestsize / 4; 550 len -= todo; 551 } else { 552 scatterwalk_map_and_copy(bf, sg, 0, todo, 0); 553 j += todo / 4; 554 len -= todo; 555 } 556 sg = sg_next(sg); 557 i++; 558 } 559 if (len > 0) { 560 dev_err(ss->dev, "remaining len %d\n", len); 561 err = -EINVAL; 562 goto theend; 563 } 564 565 if (j > 0) 566 i--; 567 568retry: 569 byte_count = areq->nbytes; 570 if (tfmctx->keylen && hmac == 0) { 571 hmac = 1; 572 /* shift all SG one slot up, to free slot 0 for IPAD */ 573 for (k = 6; k >= 0; k--) { 574 rctx->t_src[k + 1].addr = rctx->t_src[k].addr; 575 rctx->t_src[k + 1].len = rctx->t_src[k].len; 576 rctx->t_dst[k + 1].addr = rctx->t_dst[k].addr; 577 rctx->t_dst[k + 1].len = rctx->t_dst[k].len; 578 } 579 addr_xpad = dma_map_single(ss->dev, tfmctx->ipad, bs, DMA_TO_DEVICE); 580 err = dma_mapping_error(ss->dev, addr_xpad); 581 if (err) { 582 dev_err(ss->dev, "Fail to create DMA mapping of ipad\n"); 583 goto err_dma_xpad; 584 } 585 rctx->t_src[0].addr = addr_xpad; 586 rctx->t_src[0].len = bs / 4; 587 rctx->t_dst[0].addr = addr_res; 588 rctx->t_dst[0].len = digestsize / 4; 589 i++; 590 byte_count = areq->nbytes + bs; 591 } 592 if (tfmctx->keylen && hmac == 2) { 593 for (i = 0; i < MAX_SG; i++) { 594 rctx->t_src[i].addr = 0; 595 rctx->t_src[i].len = 0; 596 rctx->t_dst[i].addr = 0; 597 rctx->t_dst[i].len = 0; 598 } 599 600 addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); 601 if (dma_mapping_error(ss->dev, addr_res)) { 602 dev_err(ss->dev, "Fail to create DMA mapping of result\n"); 603 err = -EINVAL; 604 goto err_dma_result; 605 } 606 addr_xpad = dma_map_single(ss->dev, tfmctx->opad, bs, DMA_TO_DEVICE); 607 err = dma_mapping_error(ss->dev, addr_xpad); 608 if (err) { 609 dev_err(ss->dev, "Fail to create DMA mapping of opad\n"); 610 goto err_dma_xpad; 611 } 612 rctx->t_src[0].addr = addr_xpad; 613 rctx->t_src[0].len = bs / 4; 614 615 memcpy(bf, result, digestsize); 616 j = digestsize / 4; 617 i = 1; 618 byte_count = digestsize + bs; 619 620 rctx->t_dst[0].addr = addr_res; 621 rctx->t_dst[0].len = digestsize / 4; 622 } 623 624 switch (algt->ss_algo_id) { 625 case SS_ID_HASH_MD5: 626 j = hash_pad(bf, 4096, j, byte_count, true, bs); 627 break; 628 case SS_ID_HASH_SHA1: 629 case SS_ID_HASH_SHA224: 630 case SS_ID_HASH_SHA256: 631 j = hash_pad(bf, 4096, j, byte_count, false, bs); 632 break; 633 } 634 if (!j) { 635 err = -EINVAL; 636 goto theend; 637 } 638 639 addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE); 640 if (dma_mapping_error(ss->dev, addr_pad)) { 641 dev_err(ss->dev, "DMA error on padding SG\n"); 642 err = -EINVAL; 643 goto err_dma_pad; 644 } 645 rctx->t_src[i].addr = addr_pad; 646 rctx->t_src[i].len = j; 647 rctx->t_dst[i].addr = addr_res; 648 rctx->t_dst[i].len = digestsize / 4; 649 650 err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); 651 652 /* 653 * mini helper for checking dma map/unmap 654 * flow start for hmac = 0 (and HMAC = 1) 655 * HMAC = 0 656 * MAP src 657 * MAP res 658 * 659 * retry: 660 * if hmac then hmac = 1 661 * MAP xpad (ipad) 662 * if hmac == 2 663 * MAP res 664 * MAP xpad (opad) 665 * MAP pad 666 * ACTION! 667 * UNMAP pad 668 * if hmac 669 * UNMAP xpad 670 * UNMAP res 671 * if hmac < 2 672 * UNMAP SRC 673 * 674 * if hmac = 1 then hmac = 2 goto retry 675 */ 676 677 dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE); 678 679err_dma_pad: 680 if (hmac > 0) 681 dma_unmap_single(ss->dev, addr_xpad, bs, DMA_TO_DEVICE); 682err_dma_xpad: 683 dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE); 684err_dma_result: 685 if (hmac < 2) 686 dma_unmap_sg(ss->dev, areq->src, sg_nents(areq->src), 687 DMA_TO_DEVICE); 688 if (hmac == 1 && !err) { 689 hmac = 2; 690 goto retry; 691 } 692 693 if (!err) 694 memcpy(areq->result, result, crypto_ahash_digestsize(tfm)); 695theend: 696 local_bh_disable(); 697 crypto_finalize_hash_request(engine, breq, err); 698 local_bh_enable(); 699 return 0; 700} 701