1/* $OpenBSD: octcrypto.c,v 1.8 2021/10/24 10:26:22 patrick Exp $ */ 2 3/* 4 * Copyright (c) 2018 Visa Hankala 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19/* 20 * Driver for the OCTEON cryptographic unit. 21 */ 22 23#include <sys/param.h> 24#include <sys/systm.h> 25#include <sys/atomic.h> 26#include <sys/malloc.h> 27#include <sys/mbuf.h> 28#include <sys/pool.h> 29#include <sys/smr.h> 30#include <sys/tree.h> 31 32#include <crypto/cryptodev.h> 33#include <crypto/cryptosoft.h> 34#include <crypto/xform.h> 35 36#include <mips64/mips_cpu.h> 37 38#include <machine/octeonvar.h> 39 40/* Maximum number of dwords in hash IV. */ 41#define MAX_IVNW 8 42 43/* Number of dwords needed to cover `n' bytes. */ 44#define ndwords(n) (roundup(n, 8) / (8)) 45 46struct octcrypto_softc; 47 48struct octcrypto_hmac { 49 void (*transform)(const void *, size_t); 50 void (*get_iv)(uint64_t *); 51 void (*set_iv)(const uint64_t *); 52 void (*clear)(void); 53 uint16_t blocklen; 54 uint16_t taglen; 55 uint16_t countlen; 56}; 57 58struct octcrypto_session { 59 uint32_t ses_sid; /* RB key, keep first */ 60 RBT_ENTRY(octrcypto_session) 61 ses_entry; 62 struct octcrypto_softc *ses_sc; 63 struct smr_entry ses_smr; 64 65 /* AES parameters */ 66 uint64_t ses_key[4]; 67 int ses_klen; 68 uint8_t ses_nonce[AESCTR_NONCESIZE]; 69 70 /* HMAC parameters */ 71 const struct octcrypto_hmac 72 *ses_hmac; 73 uint64_t ses_iiv[MAX_IVNW]; /* HMAC inner IV */ 74 uint64_t ses_oiv[MAX_IVNW]; /* HMAC outer IV */ 75 76 /* GHASH parameters */ 77 uint64_t ses_ghkey[2]; 78 79 struct swcr_data *ses_swd; 80}; 81 82struct octcrypto_cpu { 83 uint8_t *pcpu_buf; 84 size_t pcpu_buflen; 85}; 86 87struct octcrypto_softc { 88 struct device sc_dev; 89 int32_t sc_cid; 90 uint32_t sc_sid; 91 struct mutex sc_mtx; 92 RBT_HEAD(octcrypto_tree, octcrypto_session) 93 sc_sessions; 94 struct octcrypto_cpu sc_cpu[MAXCPUS]; 95}; 96 97int octcrypto_match(struct device *, void *, void *); 98void octcrypto_attach(struct device *, struct device *, void *); 99 100int octcrypto_newsession(uint32_t *, struct cryptoini *); 101int octcrypto_freesession(uint64_t); 102int octcrypto_process(struct cryptop *); 103 104struct octcrypto_session * 105 octcrypto_get(struct octcrypto_softc *, uint32_t); 106void octcrypto_free(struct octcrypto_session *); 107void octcrypto_free_smr(void *); 108 109void octcrypto_hmac(struct cryptodesc *, uint8_t *, size_t, 110 struct octcrypto_session *, uint64_t *); 111int octcrypto_authenc_gmac(struct cryptop *, struct cryptodesc *, 112 struct cryptodesc *, struct octcrypto_session *); 113int octcrypto_authenc_hmac(struct cryptop *, struct cryptodesc *, 114 struct cryptodesc *, struct octcrypto_session *); 115int octcrypto_swauth(struct cryptop *, struct cryptodesc *, 116 struct swcr_data *, uint8_t *); 117 118void octcrypto_ghash_update_md(GHASH_CTX *, uint8_t *, size_t); 119 120void octcrypto_aes_clear(void); 121void octcrypto_aes_cbc_dec(void *, size_t, const void *); 122void octcrypto_aes_cbc_enc(void *, size_t, const void *); 123void octcrypto_aes_ctr_enc(void *, size_t, const void *); 124void octcrypto_aes_enc(uint64_t *); 125void octcrypto_aes_set_key(const uint64_t *, int); 126 127void octcrypto_ghash_finish(uint64_t *); 128void octcrypto_ghash_init(const uint64_t *, const uint64_t *); 129void octcrypto_ghash_update(const void *, size_t); 130 131void octcrypto_hash_md5(const void *, size_t); 132void octcrypto_hash_sha1(const void *, size_t); 133void octcrypto_hash_sha256(const void *, size_t); 134void octcrypto_hash_sha512(const void *, size_t); 135void octcrypto_hash_clearn(void); 136void octcrypto_hash_clearw(void); 137void octcrypto_hash_get_ivn(uint64_t *); 138void octcrypto_hash_get_ivw(uint64_t *); 139void octcrypto_hash_set_ivn(const uint64_t *); 140void octcrypto_hash_set_ivw(const uint64_t *); 141 142const struct cfattach octcrypto_ca = { 143 sizeof(struct octcrypto_softc), octcrypto_match, octcrypto_attach 144}; 145 146struct cfdriver octcrypto_cd = { 147 NULL, "octcrypto", DV_DULL 148}; 149 150static const struct octcrypto_hmac hmac_md5_96 = { 151 .transform = octcrypto_hash_md5, 152 .get_iv = octcrypto_hash_get_ivn, 153 .set_iv = octcrypto_hash_set_ivn, 154 .clear = octcrypto_hash_clearn, 155 .blocklen = 64, 156 .taglen = 12, 157 .countlen = 8 158}; 159 160static const struct octcrypto_hmac hmac_sha1_96 = { 161 .transform = octcrypto_hash_sha1, 162 .get_iv = octcrypto_hash_get_ivn, 163 .set_iv = octcrypto_hash_set_ivn, 164 .clear = octcrypto_hash_clearn, 165 .blocklen = 64, 166 .taglen = 12, 167 .countlen = 8 168}; 169 170static const struct octcrypto_hmac hmac_sha2_256_128 = { 171 .transform = octcrypto_hash_sha256, 172 .get_iv = octcrypto_hash_get_ivn, 173 .set_iv = octcrypto_hash_set_ivn, 174 .clear = octcrypto_hash_clearn, 175 .blocklen = 64, 176 .taglen = 16, 177 .countlen = 8 178}; 179 180static const struct octcrypto_hmac hmac_sha2_384_192 = { 181 .transform = octcrypto_hash_sha512, 182 .get_iv = octcrypto_hash_get_ivw, 183 .set_iv = octcrypto_hash_set_ivw, 184 .clear = octcrypto_hash_clearw, 185 .blocklen = 128, 186 .taglen = 24, 187 .countlen = 16 188}; 189 190static const struct octcrypto_hmac hmac_sha2_512_256 = { 191 .transform = octcrypto_hash_sha512, 192 .get_iv = octcrypto_hash_get_ivw, 193 .set_iv = octcrypto_hash_set_ivw, 194 .clear = octcrypto_hash_clearw, 195 .blocklen = 128, 196 .taglen = 32, 197 .countlen = 16 198}; 199 200static struct pool octcryptopl; 201static struct octcrypto_softc *octcrypto_sc; 202 203static inline int 204octcrypto_cmp(const struct octcrypto_session *a, 205 const struct octcrypto_session *b) 206{ 207 if (a->ses_sid < b->ses_sid) 208 return -1; 209 if (a->ses_sid > b->ses_sid) 210 return 1; 211 return 0; 212} 213 214RBT_PROTOTYPE(octcrypto_tree, octcrypto_session, sess_entry, octcrypto_cmp); 215RBT_GENERATE(octcrypto_tree, octcrypto_session, ses_entry, octcrypto_cmp); 216 217static inline void 218cop2_enable(void) 219{ 220 setsr(getsr() | SR_COP_2_BIT); 221} 222 223static inline void 224cop2_disable(void) 225{ 226 setsr(getsr() & ~SR_COP_2_BIT); 227} 228 229int 230octcrypto_match(struct device *parent, void *match, void *aux) 231{ 232 return 1; 233} 234 235void 236octcrypto_attach(struct device *parent, struct device *self, void *aux) 237{ 238 int algs[CRYPTO_ALGORITHM_MAX + 1]; 239 struct octcrypto_softc *sc = (struct octcrypto_softc *)self; 240 241 pool_init(&octcryptopl, sizeof(struct octcrypto_session), 0, IPL_VM, 0, 242 "octcrypto", NULL); 243 pool_setlowat(&octcryptopl, 2); 244 245 mtx_init(&sc->sc_mtx, IPL_VM); 246 RBT_INIT(octcrypto_tree, &sc->sc_sessions); 247 248 sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_MPSAFE); 249 if (sc->sc_cid < 0) { 250 printf(": could not get driver id\n"); 251 return; 252 } 253 254 printf("\n"); 255 256 memset(algs, 0, sizeof(algs)); 257 258 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED; 259 algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED; 260 algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED; 261 262 algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 263 algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 264 algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 265 algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 266 267 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 268 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 269 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 270 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 271 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 272 273 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 274 275 algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED; 276 277 octcrypto_sc = sc; 278 279 crypto_register(sc->sc_cid, algs, octcrypto_newsession, 280 octcrypto_freesession, octcrypto_process); 281 282 ghash_update = octcrypto_ghash_update_md; 283} 284 285struct octcrypto_session * 286octcrypto_get(struct octcrypto_softc *sc, uint32_t sid) 287{ 288 struct octcrypto_session *ses; 289 290 SMR_ASSERT_CRITICAL(); 291 292 mtx_enter(&sc->sc_mtx); 293 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 294 (struct octcrypto_session *)&sid); 295 mtx_leave(&sc->sc_mtx); 296 return ses; 297} 298 299void 300octcrypto_free(struct octcrypto_session *ses) 301{ 302 const struct auth_hash *axf; 303 struct swcr_data *swd; 304 305 if (ses->ses_swd != NULL) { 306 swd = ses->ses_swd; 307 axf = swd->sw_axf; 308 309 if (swd->sw_ictx != NULL) { 310 explicit_bzero(swd->sw_ictx, axf->ctxsize); 311 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize); 312 } 313 if (swd->sw_octx != NULL) { 314 explicit_bzero(swd->sw_octx, axf->ctxsize); 315 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize); 316 } 317 free(swd, M_CRYPTO_DATA, sizeof(*swd)); 318 } 319 320 explicit_bzero(ses, sizeof(*ses)); 321 pool_put(&octcryptopl, ses); 322} 323 324void 325octcrypto_free_smr(void *arg) 326{ 327 struct octcrypto_session *ses = arg; 328 329 octcrypto_free(ses); 330} 331 332int 333octcrypto_newsession(uint32_t *sidp, struct cryptoini *cri) 334{ 335 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 336 const struct auth_hash *axf; 337 struct cryptoini *c; 338 const struct octcrypto_hmac *hmac = NULL; 339 struct octcrypto_softc *sc = octcrypto_sc; 340 struct octcrypto_session *ses = NULL; 341 struct swcr_data *swd; 342 uint8_t *bptr; 343 size_t klen; 344 int i; 345 uint32_t sid; 346 347 if (sidp == NULL || cri == NULL) 348 return EINVAL; 349 350 ses = pool_get(&octcryptopl, PR_NOWAIT | PR_ZERO); 351 if (ses == NULL) 352 return ENOMEM; 353 ses->ses_sc = sc; 354 smr_init(&ses->ses_smr); 355 356 for (c = cri; c != NULL; c = c->cri_next) { 357 switch (c->cri_alg) { 358 case CRYPTO_AES_CBC: 359 ses->ses_klen = c->cri_klen / 8; 360 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 361 break; 362 363 case CRYPTO_AES_CTR: 364 case CRYPTO_AES_GCM_16: 365 case CRYPTO_AES_GMAC: 366 ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE; 367 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 368 memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen, 369 AESCTR_NONCESIZE); 370 break; 371 372 case CRYPTO_AES_128_GMAC: 373 case CRYPTO_AES_192_GMAC: 374 case CRYPTO_AES_256_GMAC: 375 cop2_enable(); 376 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 377 octcrypto_aes_enc(ses->ses_ghkey); 378 octcrypto_aes_clear(); 379 cop2_disable(); 380 break; 381 382 case CRYPTO_MD5_HMAC: 383 ses->ses_iiv[0] = 0x0123456789abcdefULL; 384 ses->ses_iiv[1] = 0xfedcba9876543210ULL; 385 ses->ses_hmac = &hmac_md5_96; 386 goto hwauthcommon; 387 388 case CRYPTO_SHA1_HMAC: 389 ses->ses_iiv[0] = 0x67452301efcdab89ULL; 390 ses->ses_iiv[1] = 0x98badcfe10325476ULL; 391 ses->ses_iiv[2] = 0xc3d2e1f000000000ULL; 392 ses->ses_hmac = &hmac_sha1_96; 393 goto hwauthcommon; 394 395 case CRYPTO_SHA2_256_HMAC: 396 ses->ses_iiv[0] = 0x6a09e667bb67ae85ULL; 397 ses->ses_iiv[1] = 0x3c6ef372a54ff53aULL; 398 ses->ses_iiv[2] = 0x510e527f9b05688cULL; 399 ses->ses_iiv[3] = 0x1f83d9ab5be0cd19ULL; 400 ses->ses_hmac = &hmac_sha2_256_128; 401 goto hwauthcommon; 402 403 case CRYPTO_SHA2_384_HMAC: 404 ses->ses_iiv[0] = 0xcbbb9d5dc1059ed8ULL; 405 ses->ses_iiv[1] = 0x629a292a367cd507ULL; 406 ses->ses_iiv[2] = 0x9159015a3070dd17ULL; 407 ses->ses_iiv[3] = 0x152fecd8f70e5939ULL; 408 ses->ses_iiv[4] = 0x67332667ffc00b31ULL; 409 ses->ses_iiv[5] = 0x8eb44a8768581511ULL; 410 ses->ses_iiv[6] = 0xdb0c2e0d64f98fa7ULL; 411 ses->ses_iiv[7] = 0x47b5481dbefa4fa4ULL; 412 ses->ses_hmac = &hmac_sha2_384_192; 413 goto hwauthcommon; 414 415 case CRYPTO_SHA2_512_HMAC: 416 ses->ses_iiv[0] = 0x6a09e667f3bcc908ULL; 417 ses->ses_iiv[1] = 0xbb67ae8584caa73bULL; 418 ses->ses_iiv[2] = 0x3c6ef372fe94f82bULL; 419 ses->ses_iiv[3] = 0xa54ff53a5f1d36f1ULL; 420 ses->ses_iiv[4] = 0x510e527fade682d1ULL; 421 ses->ses_iiv[5] = 0x9b05688c2b3e6c1fULL; 422 ses->ses_iiv[6] = 0x1f83d9abfb41bd6bULL; 423 ses->ses_iiv[7] = 0x5be0cd19137e2179ULL; 424 ses->ses_hmac = &hmac_sha2_512_256; 425 426 hwauthcommon: 427 memcpy(ses->ses_oiv, ses->ses_iiv, 428 sizeof(uint64_t) * MAX_IVNW); 429 430 bptr = (char *)block; 431 klen = c->cri_klen / 8; 432 hmac = ses->ses_hmac; 433 434 memcpy(bptr, c->cri_key, klen); 435 memset(bptr + klen, 0, hmac->blocklen - klen); 436 for (i = 0; i < hmac->blocklen; i++) 437 bptr[i] ^= HMAC_IPAD_VAL; 438 439 cop2_enable(); 440 hmac->set_iv(ses->ses_iiv); 441 hmac->transform(block, hmac->blocklen); 442 hmac->get_iv(ses->ses_iiv); 443 444 for (i = 0; i < hmac->blocklen; i++) 445 bptr[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 446 447 hmac->set_iv(ses->ses_oiv); 448 hmac->transform(block, hmac->blocklen); 449 hmac->get_iv(ses->ses_oiv); 450 hmac->clear(); 451 cop2_disable(); 452 453 explicit_bzero(block, hmac->blocklen); 454 break; 455 456 case CRYPTO_RIPEMD160_HMAC: 457 axf = &auth_hash_hmac_ripemd_160_96; 458 goto swauthcommon; 459 460 swauthcommon: 461 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 462 M_NOWAIT | M_ZERO); 463 if (swd == NULL) { 464 octcrypto_free(ses); 465 return ENOMEM; 466 } 467 ses->ses_swd = swd; 468 469 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 470 M_NOWAIT); 471 if (swd->sw_ictx == NULL) { 472 octcrypto_free(ses); 473 return ENOMEM; 474 } 475 476 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 477 M_NOWAIT); 478 if (swd->sw_octx == NULL) { 479 octcrypto_free(ses); 480 return ENOMEM; 481 } 482 483 for (i = 0; i < c->cri_klen / 8; i++) 484 c->cri_key[i] ^= HMAC_IPAD_VAL; 485 486 axf->Init(swd->sw_ictx); 487 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 488 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 489 axf->blocksize - (c->cri_klen / 8)); 490 491 for (i = 0; i < c->cri_klen / 8; i++) 492 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 493 HMAC_OPAD_VAL); 494 495 axf->Init(swd->sw_octx); 496 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 497 axf->Update(swd->sw_octx, hmac_opad_buffer, 498 axf->blocksize - (c->cri_klen / 8)); 499 500 for (i = 0; i < c->cri_klen / 8; i++) 501 c->cri_key[i] ^= HMAC_OPAD_VAL; 502 503 swd->sw_axf = axf; 504 swd->sw_alg = c->cri_alg; 505 506 break; 507 508 case CRYPTO_ESN: 509 /* nothing to do */ 510 break; 511 512 default: 513 octcrypto_free(ses); 514 return EINVAL; 515 } 516 } 517 518 mtx_enter(&sc->sc_mtx); 519 /* Find a free session ID. Assume there is one. */ 520 do { 521 sc->sc_sid++; 522 if (sc->sc_sid == 0) 523 sc->sc_sid = 1; 524 sid = sc->sc_sid; 525 } while (RBT_FIND(octcrypto_tree, &sc->sc_sessions, 526 (struct octcrypto_session *)&sid) != NULL); 527 ses->ses_sid = sid; 528 RBT_INSERT(octcrypto_tree, &sc->sc_sessions, ses); 529 mtx_leave(&sc->sc_mtx); 530 531 *sidp = ses->ses_sid; 532 return 0; 533} 534 535int 536octcrypto_freesession(uint64_t tid) 537{ 538 struct octcrypto_softc *sc = octcrypto_sc; 539 struct octcrypto_session *ses; 540 uint32_t sid = (uint32_t)tid; 541 542 mtx_enter(&sc->sc_mtx); 543 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 544 (struct octcrypto_session *)&sid); 545 if (ses != NULL) 546 RBT_REMOVE(octcrypto_tree, &sc->sc_sessions, ses); 547 mtx_leave(&sc->sc_mtx); 548 549 if (ses == NULL) 550 return EINVAL; 551 552 smr_call(&ses->ses_smr, octcrypto_free_smr, ses); 553 554 return 0; 555} 556 557enum { 558 ALG_UNHANDLED, 559 ALG_AES, 560 ALG_AES_GHASH, 561 ALG_GMAC, 562 ALG_HMAC 563}; 564 565static int 566alg_class(int alg) 567{ 568 switch (alg) { 569 case CRYPTO_AES_CBC: 570 case CRYPTO_AES_CTR: 571 return ALG_AES; 572 case CRYPTO_AES_GCM_16: 573 case CRYPTO_AES_GMAC: 574 return ALG_AES_GHASH; 575 case CRYPTO_AES_128_GMAC: 576 case CRYPTO_AES_192_GMAC: 577 case CRYPTO_AES_256_GMAC: 578 return ALG_GMAC; 579 case CRYPTO_MD5_HMAC: 580 case CRYPTO_SHA1_HMAC: 581 case CRYPTO_SHA2_256_HMAC: 582 case CRYPTO_SHA2_384_HMAC: 583 case CRYPTO_SHA2_512_HMAC: 584 return ALG_HMAC; 585 default: 586 return ALG_UNHANDLED; 587 } 588} 589 590int 591octcrypto_process(struct cryptop *crp) 592{ 593 struct cryptodesc *crd, *crd2; 594 struct octcrypto_softc *sc = octcrypto_sc; 595 struct octcrypto_session *ses = NULL; 596 int alg, alg2; 597 int error = 0; 598 int i; 599 600 KASSERT(crp->crp_ndesc >= 1); 601 602 smr_read_enter(); 603 ses = octcrypto_get(sc, (uint32_t)crp->crp_sid); 604 if (ses == NULL) { 605 error = EINVAL; 606 goto out; 607 } 608 609 if (crp->crp_ndesc == 2) { 610 crd = &crp->crp_desc[0]; 611 crd2 = &crp->crp_desc[1]; 612 alg = alg_class(crd->crd_alg); 613 alg2 = alg_class(crd2->crd_alg); 614 615 if ((alg == ALG_AES) && (alg2 == ALG_HMAC)) { 616 error = octcrypto_authenc_hmac(crp, crd, crd2, ses); 617 goto out; 618 } else if ((alg2 == ALG_AES) && (alg == ALG_HMAC)) { 619 error = octcrypto_authenc_hmac(crp, crd2, crd, ses); 620 goto out; 621 } else if ((alg == ALG_AES_GHASH) && (alg2 == ALG_GMAC)) { 622 error = octcrypto_authenc_gmac(crp, crd, crd2, ses); 623 goto out; 624 } else if ((alg2 == ALG_AES_GHASH) && (alg == ALG_GMAC)) { 625 error = octcrypto_authenc_gmac(crp, crd2, crd, ses); 626 goto out; 627 } 628 } 629 630 for (i = 0; i < crp->crp_ndesc; i++) { 631 crd = &crp->crp_desc[i]; 632 switch (crd->crd_alg) { 633 case CRYPTO_AES_CBC: 634 case CRYPTO_AES_CTR: 635 error = octcrypto_authenc_hmac(crp, crd, NULL, ses); 636 break; 637 638 case CRYPTO_MD5_HMAC: 639 case CRYPTO_SHA1_HMAC: 640 case CRYPTO_SHA2_256_HMAC: 641 case CRYPTO_SHA2_384_HMAC: 642 case CRYPTO_SHA2_512_HMAC: 643 error = octcrypto_authenc_hmac(crp, NULL, crd, ses); 644 break; 645 646 case CRYPTO_RIPEMD160_HMAC: 647 error = octcrypto_swauth(crp, crd, ses->ses_swd, 648 crp->crp_buf); 649 break; 650 651 default: 652 error = EINVAL; 653 break; 654 } 655 } 656 657out: 658 smr_read_leave(); 659 return error; 660} 661 662int 663octcrypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 664 struct swcr_data *sw, uint8_t *buf) 665{ 666 int type; 667 668 if (crp->crp_flags & CRYPTO_F_IMBUF) 669 type = CRYPTO_BUF_MBUF; 670 else 671 type = CRYPTO_BUF_IOV; 672 673 return swcr_authcompute(crp, crd, sw, buf, type); 674} 675 676int 677octcrypto_authenc_gmac(struct cryptop *crp, struct cryptodesc *crde, 678 struct cryptodesc *crda, struct octcrypto_session *ses) 679{ 680 uint64_t block[ndwords(AESCTR_BLOCKSIZE)]; 681 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 682 uint64_t iv[ndwords(AESCTR_BLOCKSIZE)]; 683 uint64_t tag[ndwords(GMAC_BLOCK_LEN)]; 684 uint8_t *buf; 685 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 686 size_t aadlen; 687 size_t ivlen = 8; 688 size_t rlen; 689 int error = 0; 690 unsigned int iskip = 0; 691 unsigned int oskip = 0; 692 693 KASSERT(crda != NULL); 694 KASSERT(crde != NULL); 695 696 rlen = roundup(crde->crd_len, AESCTR_BLOCKSIZE); 697 if (rlen > pcpu->pcpu_buflen) { 698 if (pcpu->pcpu_buf != NULL) { 699 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 700 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 701 } 702 pcpu->pcpu_buflen = 0; 703 pcpu->pcpu_buf = malloc(rlen, M_DEVBUF, M_NOWAIT | M_ZERO); 704 if (pcpu->pcpu_buf == NULL) 705 return ENOMEM; 706 pcpu->pcpu_buflen = rlen; 707 } 708 buf = pcpu->pcpu_buf; 709 710 /* Prepare the IV. */ 711 if (crde->crd_flags & CRD_F_ENCRYPT) { 712 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 713 memcpy(iv, crde->crd_iv, ivlen); 714 else 715 arc4random_buf(iv, ivlen); 716 717 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 718 if (crp->crp_flags & CRYPTO_F_IMBUF) { 719 if (m_copyback((struct mbuf *)crp->crp_buf, 720 crde->crd_inject, ivlen, (uint8_t *)iv, 721 M_NOWAIT)) { 722 error = ENOMEM; 723 goto out; 724 } 725 } else { 726 cuio_copyback((struct uio *)crp->crp_buf, 727 crde->crd_inject, ivlen, (uint8_t *)iv); 728 } 729 } 730 } else { 731 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 732 memcpy(iv, crde->crd_iv, ivlen); 733 } else { 734 if (crp->crp_flags & CRYPTO_F_IMBUF) 735 m_copydata((struct mbuf *)crp->crp_buf, 736 crde->crd_inject, ivlen, iv); 737 else 738 cuio_copydata((struct uio *)crp->crp_buf, 739 crde->crd_inject, ivlen, (uint8_t *)iv); 740 } 741 } 742 743 memset(icb, 0, sizeof(icb)); 744 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 745 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE); 746 ((uint8_t *)icb)[AESCTR_BLOCKSIZE - 1] = 1; 747 748 /* Prepare the AAD. */ 749 aadlen = crda->crd_len; 750 if (crda->crd_flags & CRD_F_ESN) { 751 aadlen += 4; 752 if (crp->crp_flags & CRYPTO_F_IMBUF) 753 m_copydata((struct mbuf *)crp->crp_buf, 754 crda->crd_skip, 4, buf); 755 else 756 cuio_copydata((struct uio *)crp->crp_buf, 757 crda->crd_skip, 4, buf); 758 memcpy(buf + 4, crda->crd_esn, 4); 759 iskip = 4; 760 oskip = 8; 761 } 762 if (crp->crp_flags & CRYPTO_F_IMBUF) 763 m_copydata((struct mbuf *)crp->crp_buf, 764 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 765 else 766 cuio_copydata((struct uio *)crp->crp_buf, 767 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 768 769 cop2_enable(); 770 octcrypto_ghash_init(ses->ses_ghkey, NULL); 771 octcrypto_ghash_update(buf, roundup(aadlen, GMAC_BLOCK_LEN)); 772 cop2_disable(); 773 774 memset(buf, 0, aadlen); 775 776 /* Copy input to the working buffer. */ 777 if (crp->crp_flags & CRYPTO_F_IMBUF) 778 m_copydata((struct mbuf *)crp->crp_buf, crde->crd_skip, 779 crde->crd_len, buf); 780 else 781 cuio_copydata((struct uio *)crp->crp_buf, crde->crd_skip, 782 crde->crd_len, buf); 783 784 cop2_enable(); 785 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 786 787 switch (crde->crd_alg) { 788 case CRYPTO_AES_GCM_16: 789 if (crde->crd_flags & CRD_F_ENCRYPT) { 790 octcrypto_aes_ctr_enc(buf, rlen, icb); 791 memset(buf + crde->crd_len, 0, rlen - crde->crd_len); 792 octcrypto_ghash_update(buf, rlen); 793 } else { 794 octcrypto_ghash_update(buf, rlen); 795 octcrypto_aes_ctr_enc(buf, rlen, icb); 796 } 797 break; 798 799 case CRYPTO_AES_GMAC: 800 octcrypto_ghash_update(buf, rlen); 801 break; 802 } 803 804 block[0] = htobe64(aadlen * 8); 805 block[1] = htobe64(crde->crd_len * 8); 806 octcrypto_ghash_update(block, GMAC_BLOCK_LEN); 807 octcrypto_ghash_finish(tag); 808 809 block[0] = icb[0]; 810 block[1] = icb[1]; 811 octcrypto_aes_enc(block); 812 tag[0] ^= block[0]; 813 tag[1] ^= block[1]; 814 815 octcrypto_aes_clear(); 816 cop2_disable(); 817 818 /* Copy back the output. */ 819 if (crp->crp_flags & CRYPTO_F_IMBUF) { 820 if (m_copyback((struct mbuf *)crp->crp_buf, 821 crde->crd_skip, crde->crd_len, buf, M_NOWAIT)) { 822 error = ENOMEM; 823 goto out; 824 } 825 } else { 826 cuio_copyback((struct uio *)crp->crp_buf, 827 crde->crd_skip, crde->crd_len, buf); 828 } 829 830 /* Copy back the authentication tag. */ 831 if (crp->crp_flags & CRYPTO_F_IMBUF) { 832 if (m_copyback((struct mbuf *)crp->crp_buf, crda->crd_inject, 833 GMAC_DIGEST_LEN, tag, M_NOWAIT)) { 834 error = ENOMEM; 835 goto out; 836 } 837 } else { 838 memcpy(crp->crp_mac, tag, GMAC_DIGEST_LEN); 839 } 840 841out: 842 explicit_bzero(buf, rlen); 843 explicit_bzero(icb, sizeof(icb)); 844 explicit_bzero(tag, sizeof(tag)); 845 846 return error; 847} 848 849void 850octcrypto_hmac(struct cryptodesc *crda, uint8_t *buf, size_t len, 851 struct octcrypto_session *ses, uint64_t *res) 852{ 853 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 854 uint8_t *bptr = (uint8_t *)block; 855 const struct octcrypto_hmac *hmac = ses->ses_hmac; 856 size_t left; 857 858 cop2_enable(); 859 860 /* 861 * Compute the inner hash. 862 */ 863 864 hmac->set_iv(ses->ses_iiv); 865 hmac->transform(buf, len); 866 867 memset(block, 0, hmac->blocklen); 868 left = len & (hmac->blocklen - 1); 869 bptr[left] = 0x80; 870 if (left > 0) { 871 memcpy(block, buf + len - left, left); 872 873 if (roundup(left + 1, hmac->countlen) > 874 (hmac->blocklen - hmac->countlen)) { 875 hmac->transform(block, hmac->blocklen); 876 memset(block, 0, hmac->blocklen); 877 } 878 } 879 880 switch (crda->crd_alg) { 881 case CRYPTO_MD5_HMAC: 882 block[7] = htole64((64 + len) * 8); 883 break; 884 case CRYPTO_SHA1_HMAC: 885 case CRYPTO_SHA2_256_HMAC: 886 block[7] = htobe64((64 + len) * 8); 887 break; 888 case CRYPTO_SHA2_384_HMAC: 889 case CRYPTO_SHA2_512_HMAC: 890 block[15] = htobe64((128 + len) * 8); 891 break; 892 } 893 894 hmac->transform(block, hmac->blocklen); 895 896 /* 897 * Compute the outer hash. 898 */ 899 900 memset(block, 0, hmac->blocklen); 901 hmac->get_iv(block); 902 hmac->set_iv(ses->ses_oiv); 903 904 switch (crda->crd_alg) { 905 case CRYPTO_MD5_HMAC: 906 block[2] = htobe64(1ULL << 63); 907 block[7] = htole64((64 + 16) * 8); 908 break; 909 case CRYPTO_SHA1_HMAC: 910 block[2] |= htobe64(1ULL << 31); 911 block[7] = htobe64((64 + 20) * 8); 912 break; 913 case CRYPTO_SHA2_256_HMAC: 914 block[4] = htobe64(1ULL << 63); 915 block[7] = htobe64((64 + 32) * 8); 916 break; 917 case CRYPTO_SHA2_384_HMAC: 918 /* 919 * The computed digest is 512 bits long. 920 * It has to be truncated to 384 bits. 921 */ 922 block[6] = htobe64(1ULL << 63); 923 block[7] = 0; /* truncation */ 924 block[15] = htobe64((128 + 48) * 8); 925 break; 926 case CRYPTO_SHA2_512_HMAC: 927 block[8] = htobe64(1ULL << 63); 928 block[15] = htobe64((128 + 64) * 8); 929 break; 930 } 931 932 hmac->transform(block, hmac->blocklen); 933 hmac->get_iv(res); 934 hmac->clear(); 935 936 cop2_disable(); 937 938 explicit_bzero(block, sizeof(block)); 939} 940 941int 942octcrypto_authenc_hmac(struct cryptop *crp, struct cryptodesc *crde, 943 struct cryptodesc *crda, struct octcrypto_session *ses) 944{ 945 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 946 uint64_t iv[ndwords(EALG_MAX_BLOCK_LEN)]; 947 uint64_t tag[ndwords(AALG_MAX_RESULT_LEN)]; 948 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 949 uint8_t *buf, *authbuf, *encbuf; 950 size_t authlen; 951 size_t buflen; 952 size_t len; 953 size_t skip; 954 off_t authskip = 0; 955 off_t encskip = 0; 956 int error = 0; 957 int ivlen; 958 959 if (crde != NULL && crda != NULL) { 960 skip = MIN(crde->crd_skip, crda->crd_skip); 961 len = MAX(crde->crd_skip + crde->crd_len, 962 crda->crd_skip + crda->crd_len) - skip; 963 964 if (crda->crd_skip < crde->crd_skip) 965 encskip = crde->crd_skip - crda->crd_skip; 966 else 967 authskip = crda->crd_skip - crde->crd_skip; 968 } else if (crde != NULL) { 969 skip = crde->crd_skip; 970 len = crde->crd_len; 971 } else { 972 KASSERT(crda != NULL); 973 974 skip = crda->crd_skip; 975 len = crda->crd_len; 976 } 977 978 buflen = len; 979 980 /* Reserve space for ESN. */ 981 if (crda != NULL && (crda->crd_flags & CRD_F_ESN) != 0) 982 buflen += 4; 983 984 buflen = roundup(buflen, EALG_MAX_BLOCK_LEN); 985 if (buflen > pcpu->pcpu_buflen) { 986 if (pcpu->pcpu_buf != NULL) { 987 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 988 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 989 } 990 pcpu->pcpu_buflen = 0; 991 pcpu->pcpu_buf = malloc(buflen, M_DEVBUF, M_NOWAIT | M_ZERO); 992 if (pcpu->pcpu_buf == NULL) 993 return ENOMEM; 994 pcpu->pcpu_buflen = buflen; 995 } 996 buf = pcpu->pcpu_buf; 997 998 authbuf = buf + authskip; 999 encbuf = buf + encskip; 1000 1001 /* Prepare the IV. */ 1002 if (crde != NULL) { 1003 /* CBC uses 16 bytes, CTR 8 bytes. */ 1004 ivlen = (crde->crd_alg == CRYPTO_AES_CBC) ? 16 : 8; 1005 1006 if (crde->crd_flags & CRD_F_ENCRYPT) { 1007 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 1008 memcpy(iv, crde->crd_iv, ivlen); 1009 else 1010 arc4random_buf(iv, ivlen); 1011 1012 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 1013 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1014 if (m_copyback( 1015 (struct mbuf *)crp->crp_buf, 1016 crde->crd_inject, ivlen, iv, 1017 M_NOWAIT)) { 1018 error = ENOMEM; 1019 goto out; 1020 } 1021 } else { 1022 cuio_copyback( 1023 (struct uio *)crp->crp_buf, 1024 crde->crd_inject, ivlen, iv); 1025 } 1026 } 1027 } else { 1028 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 1029 memcpy(iv, crde->crd_iv, ivlen); 1030 } else { 1031 if (crp->crp_flags & CRYPTO_F_IMBUF) 1032 m_copydata((struct mbuf *)crp->crp_buf, 1033 crde->crd_inject, ivlen, iv); 1034 else 1035 cuio_copydata( 1036 (struct uio *)crp->crp_buf, 1037 crde->crd_inject, ivlen, 1038 (uint8_t *)iv); 1039 } 1040 } 1041 } 1042 1043 /* Copy input to the working buffer. */ 1044 if (crp->crp_flags & CRYPTO_F_IMBUF) 1045 m_copydata((struct mbuf *)crp->crp_buf, skip, len, buf); 1046 else 1047 cuio_copydata((struct uio *)crp->crp_buf, skip, len, buf); 1048 1049 /* If ESN is used, append it to the buffer. */ 1050 if (crda != NULL) { 1051 authlen = crda->crd_len; 1052 if (crda->crd_flags & CRD_F_ESN) { 1053 memcpy(buf + len, crda->crd_esn, 4); 1054 authlen += 4; 1055 } 1056 } 1057 1058 if (crde != NULL) { 1059 /* Compute authentication tag before decryption. */ 1060 if (crda != NULL && (crde->crd_flags & CRD_F_ENCRYPT) == 0) 1061 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1062 1063 /* Apply the cipher. */ 1064 switch (crde->crd_alg) { 1065 case CRYPTO_AES_CBC: 1066 cop2_enable(); 1067 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1068 if (crde->crd_flags & CRD_F_ENCRYPT) 1069 octcrypto_aes_cbc_enc(encbuf, crde->crd_len, 1070 iv); 1071 else 1072 octcrypto_aes_cbc_dec(encbuf, crde->crd_len, 1073 iv); 1074 octcrypto_aes_clear(); 1075 cop2_disable(); 1076 break; 1077 1078 case CRYPTO_AES_CTR: 1079 memset(icb, 0, sizeof(icb)); 1080 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 1081 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, 1082 AESCTR_IVSIZE); 1083 cop2_enable(); 1084 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1085 octcrypto_aes_ctr_enc(encbuf, crde->crd_len, icb); 1086 octcrypto_aes_clear(); 1087 cop2_disable(); 1088 explicit_bzero(icb, sizeof(icb)); 1089 break; 1090 } 1091 1092 /* Copy back the output. */ 1093 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1094 if (m_copyback((struct mbuf *)crp->crp_buf, 1095 crde->crd_skip, crde->crd_len, encbuf, M_NOWAIT)) { 1096 error = ENOMEM; 1097 goto out; 1098 } 1099 } else { 1100 cuio_copyback((struct uio *)crp->crp_buf, 1101 crde->crd_skip, crde->crd_len, encbuf); 1102 } 1103 } 1104 1105 if (crda != NULL) { 1106 /* 1107 * Compute authentication tag after encryption. 1108 * This also handles the authentication only case. 1109 */ 1110 if (crde == NULL || (crde->crd_flags & CRD_F_ENCRYPT) != 0) 1111 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1112 1113 /* Copy back the authentication tag. */ 1114 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1115 if (m_copyback((struct mbuf *)crp->crp_buf, 1116 crda->crd_inject, ses->ses_hmac->taglen, tag, 1117 M_NOWAIT)) { 1118 error = ENOMEM; 1119 goto out; 1120 } 1121 } else { 1122 memcpy(crp->crp_mac, tag, ses->ses_hmac->taglen); 1123 } 1124 1125 explicit_bzero(tag, sizeof(tag)); 1126 } 1127 1128out: 1129 explicit_bzero(buf, len); 1130 return error; 1131} 1132 1133void 1134octcrypto_ghash_update_md(GHASH_CTX *ghash, uint8_t *src, size_t len) 1135{ 1136 CTASSERT(offsetof(GHASH_CTX, H) % 8 == 0); 1137 CTASSERT(offsetof(GHASH_CTX, S) % 8 == 0); 1138 1139 cop2_enable(); 1140 octcrypto_ghash_init((uint64_t *)ghash->H, (uint64_t *)ghash->S); 1141 octcrypto_ghash_update(src, len); 1142 octcrypto_ghash_finish((uint64_t *)ghash->S); 1143 cop2_disable(); 1144} 1145