1210409Skib/*- 2210409Skib * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3210409Skib * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> 4275732Sjmg * Copyright (c) 2014 The FreeBSD Foundation 5210409Skib * All rights reserved. 6210409Skib * 7275732Sjmg * Portions of this software were developed by John-Mark Gurney 8275732Sjmg * under sponsorship of the FreeBSD Foundation and 9275732Sjmg * Rubicon Communications, LLC (Netgate). 10275732Sjmg * 11210409Skib * Redistribution and use in source and binary forms, with or without 12210409Skib * modification, are permitted provided that the following conditions 13210409Skib * are met: 14210409Skib * 1. Redistributions of source code must retain the above copyright 15210409Skib * notice, this list of conditions and the following disclaimer. 16210409Skib * 2. Redistributions in binary form must reproduce the above copyright 17210409Skib * notice, this list of conditions and the following disclaimer in the 18210409Skib * documentation and/or other materials provided with the distribution. 19210409Skib * 20210409Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 21210409Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22210409Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23210409Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 24210409Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25210409Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26210409Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27210409Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28210409Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29210409Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30210409Skib * SUCH DAMAGE. 31210409Skib */ 32210409Skib 33210409Skib#include <sys/cdefs.h> 34210409Skib__FBSDID("$FreeBSD$"); 35210409Skib 36210409Skib#include <sys/param.h> 37210409Skib#include <sys/systm.h> 38210409Skib#include <sys/kernel.h> 39210409Skib#include <sys/kobj.h> 40210409Skib#include <sys/libkern.h> 41210409Skib#include <sys/lock.h> 42210409Skib#include <sys/module.h> 43210409Skib#include <sys/malloc.h> 44210409Skib#include <sys/rwlock.h> 45210409Skib#include <sys/bus.h> 46210409Skib#include <sys/uio.h> 47275732Sjmg#include <sys/mbuf.h> 48285289Sjmg#include <sys/smp.h> 49210409Skib#include <crypto/aesni/aesni.h> 50255187Sjmg#include <cryptodev_if.h> 51275732Sjmg#include <opencrypto/gmac.h> 52210409Skib 53285289Sjmgstatic struct mtx_padalign *ctx_mtx; 54285289Sjmgstatic struct fpu_kern_ctx **ctx_fpu; 55285289Sjmg 56210409Skibstruct aesni_softc { 57285289Sjmg int dieing; 58210409Skib int32_t cid; 59210409Skib uint32_t sid; 60210409Skib TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions; 61210409Skib struct rwlock lock; 62210409Skib}; 63210409Skib 64285289Sjmg#define AQUIRE_CTX(i, ctx) \ 65285289Sjmg do { \ 66285289Sjmg (i) = PCPU_GET(cpuid); \ 67285289Sjmg mtx_lock(&ctx_mtx[(i)]); \ 68285289Sjmg (ctx) = ctx_fpu[(i)]; \ 69285289Sjmg } while (0) 70285289Sjmg#define RELEASE_CTX(i, ctx) \ 71285289Sjmg do { \ 72285289Sjmg mtx_unlock(&ctx_mtx[(i)]); \ 73285289Sjmg (i) = -1; \ 74285289Sjmg (ctx) = NULL; \ 75285289Sjmg } while (0) 76285289Sjmg 77210409Skibstatic int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri); 78210409Skibstatic int aesni_freesession(device_t, uint64_t tid); 79210409Skibstatic void aesni_freesession_locked(struct aesni_softc *sc, 80210409Skib struct aesni_session *ses); 81267815Skibstatic int aesni_cipher_setup(struct aesni_session *ses, 82267815Skib struct cryptoini *encini); 83267815Skibstatic int aesni_cipher_process(struct aesni_session *ses, 84275732Sjmg struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp); 85210409Skib 86210409SkibMALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data"); 87210409Skib 88210409Skibstatic void 89210409Skibaesni_identify(driver_t *drv, device_t parent) 90210409Skib{ 91210409Skib 92210409Skib /* NB: order 10 is so we get attached after h/w devices */ 93210409Skib if (device_find_child(parent, "aesni", -1) == NULL && 94210409Skib BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0) 95210409Skib panic("aesni: could not attach"); 96210409Skib} 97210409Skib 98210409Skibstatic int 99210409Skibaesni_probe(device_t dev) 100210409Skib{ 101210409Skib 102210409Skib if ((cpu_feature2 & CPUID2_AESNI) == 0) { 103210409Skib device_printf(dev, "No AESNI support.\n"); 104210409Skib return (EINVAL); 105210409Skib } 106255187Sjmg 107275732Sjmg if ((cpu_feature2 & CPUID2_SSE41) == 0) { 108275732Sjmg device_printf(dev, "No SSE4.1 support.\n"); 109255187Sjmg return (EINVAL); 110255187Sjmg } 111255187Sjmg 112275732Sjmg device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM"); 113210409Skib return (0); 114210409Skib} 115210409Skib 116285289Sjmgstatic void 117285289Sjmgaensi_cleanctx(void) 118285289Sjmg{ 119285289Sjmg int i; 120285289Sjmg 121285289Sjmg /* XXX - no way to return driverid */ 122285289Sjmg CPU_FOREACH(i) { 123285289Sjmg if (ctx_fpu[i] != NULL) { 124285289Sjmg mtx_destroy(&ctx_mtx[i]); 125285289Sjmg fpu_kern_free_ctx(ctx_fpu[i]); 126285289Sjmg } 127285289Sjmg ctx_fpu[i] = NULL; 128285289Sjmg } 129285289Sjmg free(ctx_mtx, M_AESNI); 130285289Sjmg ctx_mtx = NULL; 131285289Sjmg free(ctx_fpu, M_AESNI); 132285289Sjmg ctx_fpu = NULL; 133285289Sjmg} 134285289Sjmg 135210409Skibstatic int 136210409Skibaesni_attach(device_t dev) 137210409Skib{ 138210409Skib struct aesni_softc *sc; 139285289Sjmg int i; 140210409Skib 141210409Skib sc = device_get_softc(dev); 142285289Sjmg sc->dieing = 0; 143210409Skib TAILQ_INIT(&sc->sessions); 144210409Skib sc->sid = 1; 145285289Sjmg 146258492Sjmg sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | 147258492Sjmg CRYPTOCAP_F_SYNC); 148210409Skib if (sc->cid < 0) { 149210409Skib device_printf(dev, "Could not get crypto driver id.\n"); 150210409Skib return (ENOMEM); 151210409Skib } 152210409Skib 153285289Sjmg ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI, 154285289Sjmg M_WAITOK|M_ZERO); 155285289Sjmg ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI, 156285289Sjmg M_WAITOK|M_ZERO); 157285289Sjmg 158285289Sjmg CPU_FOREACH(i) { 159285289Sjmg ctx_fpu[i] = fpu_kern_alloc_ctx(0); 160285289Sjmg mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW); 161285289Sjmg } 162285289Sjmg 163210409Skib rw_init(&sc->lock, "aesni_lock"); 164210409Skib crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); 165275732Sjmg crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); 166275732Sjmg crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); 167275732Sjmg crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); 168275732Sjmg crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); 169275732Sjmg crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); 170213069Spjd crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); 171210409Skib return (0); 172210409Skib} 173210409Skib 174210409Skibstatic int 175210409Skibaesni_detach(device_t dev) 176210409Skib{ 177210409Skib struct aesni_softc *sc; 178210409Skib struct aesni_session *ses; 179210409Skib 180210409Skib sc = device_get_softc(dev); 181285289Sjmg 182210409Skib rw_wlock(&sc->lock); 183210409Skib TAILQ_FOREACH(ses, &sc->sessions, next) { 184210409Skib if (ses->used) { 185210409Skib rw_wunlock(&sc->lock); 186210409Skib device_printf(dev, 187210409Skib "Cannot detach, sessions still active.\n"); 188210409Skib return (EBUSY); 189210409Skib } 190210409Skib } 191285289Sjmg sc->dieing = 1; 192210409Skib while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) { 193210409Skib TAILQ_REMOVE(&sc->sessions, ses, next); 194210409Skib free(ses, M_AESNI); 195210409Skib } 196210409Skib rw_wunlock(&sc->lock); 197285289Sjmg crypto_unregister_all(sc->cid); 198285289Sjmg 199210409Skib rw_destroy(&sc->lock); 200285289Sjmg 201285289Sjmg aensi_cleanctx(); 202285289Sjmg 203210409Skib return (0); 204210409Skib} 205210409Skib 206210409Skibstatic int 207210409Skibaesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 208210409Skib{ 209210409Skib struct aesni_softc *sc; 210210409Skib struct aesni_session *ses; 211210409Skib struct cryptoini *encini; 212210409Skib int error; 213210409Skib 214275732Sjmg if (sidp == NULL || cri == NULL) { 215275732Sjmg CRYPTDEB("no sidp or cri"); 216210409Skib return (EINVAL); 217275732Sjmg } 218210409Skib 219210409Skib sc = device_get_softc(dev); 220285289Sjmg if (sc->dieing) 221285289Sjmg return (EINVAL); 222285289Sjmg 223210409Skib ses = NULL; 224210409Skib encini = NULL; 225210409Skib for (; cri != NULL; cri = cri->cri_next) { 226210409Skib switch (cri->cri_alg) { 227210409Skib case CRYPTO_AES_CBC: 228275732Sjmg case CRYPTO_AES_ICM: 229213069Spjd case CRYPTO_AES_XTS: 230275732Sjmg case CRYPTO_AES_NIST_GCM_16: 231275732Sjmg if (encini != NULL) { 232275732Sjmg CRYPTDEB("encini already set"); 233210409Skib return (EINVAL); 234275732Sjmg } 235210409Skib encini = cri; 236210409Skib break; 237275732Sjmg case CRYPTO_AES_128_NIST_GMAC: 238275732Sjmg case CRYPTO_AES_192_NIST_GMAC: 239275732Sjmg case CRYPTO_AES_256_NIST_GMAC: 240275732Sjmg /* 241275732Sjmg * nothing to do here, maybe in the future cache some 242275732Sjmg * values for GHASH 243275732Sjmg */ 244275732Sjmg break; 245210409Skib default: 246275732Sjmg CRYPTDEB("unhandled algorithm"); 247210409Skib return (EINVAL); 248210409Skib } 249210409Skib } 250275732Sjmg if (encini == NULL) { 251275732Sjmg CRYPTDEB("no cipher"); 252210409Skib return (EINVAL); 253275732Sjmg } 254210409Skib 255210409Skib rw_wlock(&sc->lock); 256285289Sjmg if (sc->dieing) { 257285289Sjmg rw_wunlock(&sc->lock); 258285289Sjmg return (EINVAL); 259285289Sjmg } 260210409Skib /* 261210409Skib * Free sessions goes first, so if first session is used, we need to 262210409Skib * allocate one. 263210409Skib */ 264210409Skib ses = TAILQ_FIRST(&sc->sessions); 265210409Skib if (ses == NULL || ses->used) { 266210409Skib ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO); 267210409Skib if (ses == NULL) { 268210409Skib rw_wunlock(&sc->lock); 269210409Skib return (ENOMEM); 270210409Skib } 271210409Skib ses->id = sc->sid++; 272210409Skib } else { 273210409Skib TAILQ_REMOVE(&sc->sessions, ses, next); 274210409Skib } 275210409Skib ses->used = 1; 276210409Skib TAILQ_INSERT_TAIL(&sc->sessions, ses, next); 277210409Skib rw_wunlock(&sc->lock); 278213069Spjd ses->algo = encini->cri_alg; 279210409Skib 280210409Skib error = aesni_cipher_setup(ses, encini); 281210409Skib if (error != 0) { 282275732Sjmg CRYPTDEB("setup failed"); 283210409Skib rw_wlock(&sc->lock); 284210409Skib aesni_freesession_locked(sc, ses); 285210409Skib rw_wunlock(&sc->lock); 286210409Skib return (error); 287210409Skib } 288210409Skib 289210409Skib *sidp = ses->id; 290210409Skib return (0); 291210409Skib} 292210409Skib 293210409Skibstatic void 294210409Skibaesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses) 295210409Skib{ 296210409Skib uint32_t sid; 297210409Skib 298285289Sjmg rw_assert(&sc->lock, RA_WLOCKED); 299285289Sjmg 300210409Skib sid = ses->id; 301210409Skib TAILQ_REMOVE(&sc->sessions, ses, next); 302275732Sjmg *ses = (struct aesni_session){}; 303210409Skib ses->id = sid; 304210409Skib TAILQ_INSERT_HEAD(&sc->sessions, ses, next); 305210409Skib} 306210409Skib 307210409Skibstatic int 308210409Skibaesni_freesession(device_t dev, uint64_t tid) 309210409Skib{ 310210409Skib struct aesni_softc *sc; 311210409Skib struct aesni_session *ses; 312210409Skib uint32_t sid; 313210409Skib 314210409Skib sc = device_get_softc(dev); 315210409Skib sid = ((uint32_t)tid) & 0xffffffff; 316210409Skib rw_wlock(&sc->lock); 317210409Skib TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) { 318210409Skib if (ses->id == sid) 319210409Skib break; 320210409Skib } 321210409Skib if (ses == NULL) { 322210409Skib rw_wunlock(&sc->lock); 323210409Skib return (EINVAL); 324210409Skib } 325210409Skib aesni_freesession_locked(sc, ses); 326210409Skib rw_wunlock(&sc->lock); 327210409Skib return (0); 328210409Skib} 329210409Skib 330210409Skibstatic int 331210409Skibaesni_process(device_t dev, struct cryptop *crp, int hint __unused) 332210409Skib{ 333210409Skib struct aesni_softc *sc = device_get_softc(dev); 334210409Skib struct aesni_session *ses = NULL; 335275732Sjmg struct cryptodesc *crd, *enccrd, *authcrd; 336275732Sjmg int error, needauth; 337210409Skib 338210409Skib error = 0; 339210409Skib enccrd = NULL; 340275732Sjmg authcrd = NULL; 341275732Sjmg needauth = 0; 342210409Skib 343210409Skib /* Sanity check. */ 344210409Skib if (crp == NULL) 345210409Skib return (EINVAL); 346210409Skib 347210409Skib if (crp->crp_callback == NULL || crp->crp_desc == NULL) { 348210409Skib error = EINVAL; 349210409Skib goto out; 350210409Skib } 351210409Skib 352210409Skib for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { 353210409Skib switch (crd->crd_alg) { 354210409Skib case CRYPTO_AES_CBC: 355275732Sjmg case CRYPTO_AES_ICM: 356213069Spjd case CRYPTO_AES_XTS: 357210409Skib if (enccrd != NULL) { 358210409Skib error = EINVAL; 359210409Skib goto out; 360210409Skib } 361210409Skib enccrd = crd; 362210409Skib break; 363275732Sjmg 364275732Sjmg case CRYPTO_AES_NIST_GCM_16: 365275732Sjmg if (enccrd != NULL) { 366275732Sjmg error = EINVAL; 367275732Sjmg goto out; 368275732Sjmg } 369275732Sjmg enccrd = crd; 370275732Sjmg needauth = 1; 371275732Sjmg break; 372275732Sjmg 373275732Sjmg case CRYPTO_AES_128_NIST_GMAC: 374275732Sjmg case CRYPTO_AES_192_NIST_GMAC: 375275732Sjmg case CRYPTO_AES_256_NIST_GMAC: 376275732Sjmg if (authcrd != NULL) { 377275732Sjmg error = EINVAL; 378275732Sjmg goto out; 379275732Sjmg } 380275732Sjmg authcrd = crd; 381275732Sjmg needauth = 1; 382275732Sjmg break; 383275732Sjmg 384210409Skib default: 385275732Sjmg error = EINVAL; 386275732Sjmg goto out; 387210409Skib } 388210409Skib } 389275732Sjmg 390275732Sjmg if (enccrd == NULL || (needauth && authcrd == NULL)) { 391210409Skib error = EINVAL; 392210409Skib goto out; 393210409Skib } 394210409Skib 395275732Sjmg /* CBC & XTS can only handle full blocks for now */ 396275732Sjmg if ((enccrd->crd_alg == CRYPTO_AES_CBC || enccrd->crd_alg == 397275732Sjmg CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0) { 398275732Sjmg error = EINVAL; 399275732Sjmg goto out; 400275732Sjmg } 401275732Sjmg 402210409Skib rw_rlock(&sc->lock); 403210409Skib TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) { 404210409Skib if (ses->id == (crp->crp_sid & 0xffffffff)) 405210409Skib break; 406210409Skib } 407210409Skib rw_runlock(&sc->lock); 408210409Skib if (ses == NULL) { 409210409Skib error = EINVAL; 410210409Skib goto out; 411210409Skib } 412210409Skib 413275732Sjmg error = aesni_cipher_process(ses, enccrd, authcrd, crp); 414210409Skib if (error != 0) 415210409Skib goto out; 416210409Skib 417210409Skibout: 418210409Skib crp->crp_etype = error; 419210409Skib crypto_done(crp); 420210409Skib return (error); 421210409Skib} 422210409Skib 423210409Skibuint8_t * 424210409Skibaesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, 425210409Skib int *allocated) 426210409Skib{ 427275732Sjmg struct mbuf *m; 428210409Skib struct uio *uio; 429210409Skib struct iovec *iov; 430210409Skib uint8_t *addr; 431210409Skib 432275732Sjmg if (crp->crp_flags & CRYPTO_F_IMBUF) { 433275732Sjmg m = (struct mbuf *)crp->crp_buf; 434275732Sjmg if (m->m_next != NULL) 435275732Sjmg goto alloc; 436275732Sjmg addr = mtod(m, uint8_t *); 437275732Sjmg } else if (crp->crp_flags & CRYPTO_F_IOV) { 438210409Skib uio = (struct uio *)crp->crp_buf; 439210409Skib if (uio->uio_iovcnt != 1) 440210409Skib goto alloc; 441210409Skib iov = uio->uio_iov; 442275732Sjmg addr = (uint8_t *)iov->iov_base; 443210409Skib } else 444275732Sjmg addr = (uint8_t *)crp->crp_buf; 445210409Skib *allocated = 0; 446275732Sjmg addr += enccrd->crd_skip; 447210409Skib return (addr); 448210409Skib 449210409Skiballoc: 450210409Skib addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT); 451210409Skib if (addr != NULL) { 452210409Skib *allocated = 1; 453210409Skib crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 454210409Skib enccrd->crd_len, addr); 455210409Skib } else 456210409Skib *allocated = 0; 457210409Skib return (addr); 458210409Skib} 459210409Skib 460210409Skibstatic device_method_t aesni_methods[] = { 461210409Skib DEVMETHOD(device_identify, aesni_identify), 462210409Skib DEVMETHOD(device_probe, aesni_probe), 463210409Skib DEVMETHOD(device_attach, aesni_attach), 464210409Skib DEVMETHOD(device_detach, aesni_detach), 465210409Skib 466210409Skib DEVMETHOD(cryptodev_newsession, aesni_newsession), 467210409Skib DEVMETHOD(cryptodev_freesession, aesni_freesession), 468210409Skib DEVMETHOD(cryptodev_process, aesni_process), 469210409Skib 470210409Skib {0, 0}, 471210409Skib}; 472210409Skib 473210409Skibstatic driver_t aesni_driver = { 474210409Skib "aesni", 475210409Skib aesni_methods, 476210409Skib sizeof(struct aesni_softc), 477210409Skib}; 478210409Skibstatic devclass_t aesni_devclass; 479210409Skib 480210409SkibDRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0); 481210409SkibMODULE_VERSION(aesni, 1); 482210409SkibMODULE_DEPEND(aesni, crypto, 1, 1, 1); 483267815Skib 484267815Skibstatic int 485267815Skibaesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) 486267815Skib{ 487285289Sjmg struct fpu_kern_ctx *ctx; 488267815Skib int error; 489285289Sjmg int kt, ctxidx; 490267815Skib 491285289Sjmg kt = is_fpu_kern_thread(0); 492285289Sjmg if (!kt) { 493285289Sjmg AQUIRE_CTX(ctxidx, ctx); 494285289Sjmg error = fpu_kern_enter(curthread, ctx, 495285289Sjmg FPU_KERN_NORMAL | FPU_KERN_KTHR); 496285289Sjmg if (error != 0) 497285289Sjmg goto out; 498285289Sjmg } 499285289Sjmg 500267815Skib error = aesni_cipher_setup_common(ses, encini->cri_key, 501267815Skib encini->cri_klen); 502285289Sjmg 503285289Sjmg if (!kt) { 504285289Sjmg fpu_kern_leave(curthread, ctx); 505285289Sjmgout: 506285289Sjmg RELEASE_CTX(ctxidx, ctx); 507285289Sjmg } 508267815Skib return (error); 509267815Skib} 510267815Skib 511275732Sjmg/* 512275732Sjmg * authcrd contains the associated date. 513275732Sjmg */ 514267815Skibstatic int 515267815Skibaesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, 516275732Sjmg struct cryptodesc *authcrd, struct cryptop *crp) 517267815Skib{ 518285289Sjmg struct fpu_kern_ctx *ctx; 519285216Sjmg uint8_t iv[AES_BLOCK_LEN]; 520275732Sjmg uint8_t tag[GMAC_DIGEST_LEN]; 521275732Sjmg uint8_t *buf, *authbuf; 522275732Sjmg int error, allocated, authallocated; 523275732Sjmg int ivlen, encflag; 524285289Sjmg int kt, ctxidx; 525267815Skib 526275732Sjmg encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; 527275732Sjmg 528275732Sjmg if ((enccrd->crd_alg == CRYPTO_AES_ICM || 529275732Sjmg enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) && 530275732Sjmg (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) 531275732Sjmg return (EINVAL); 532275732Sjmg 533267815Skib buf = aesni_cipher_alloc(enccrd, crp, &allocated); 534267815Skib if (buf == NULL) 535267815Skib return (ENOMEM); 536267815Skib 537298332Scem error = 0; 538275732Sjmg authbuf = NULL; 539275732Sjmg authallocated = 0; 540275732Sjmg if (authcrd != NULL) { 541275732Sjmg authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); 542275732Sjmg if (authbuf == NULL) { 543275732Sjmg error = ENOMEM; 544275732Sjmg goto out1; 545275732Sjmg } 546275732Sjmg } 547275732Sjmg 548285289Sjmg kt = is_fpu_kern_thread(0); 549285289Sjmg if (!kt) { 550285289Sjmg AQUIRE_CTX(ctxidx, ctx); 551285289Sjmg error = fpu_kern_enter(curthread, ctx, 552285289Sjmg FPU_KERN_NORMAL|FPU_KERN_KTHR); 553285289Sjmg if (error != 0) 554285289Sjmg goto out2; 555285289Sjmg } 556267815Skib 557267815Skib if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 558267815Skib error = aesni_cipher_setup_common(ses, enccrd->crd_key, 559267815Skib enccrd->crd_klen); 560267815Skib if (error != 0) 561267815Skib goto out; 562267815Skib } 563267815Skib 564275732Sjmg /* XXX - validate that enccrd and authcrd have/use same key? */ 565275732Sjmg switch (enccrd->crd_alg) { 566275732Sjmg case CRYPTO_AES_CBC: 567275732Sjmg case CRYPTO_AES_ICM: 568275732Sjmg ivlen = AES_BLOCK_LEN; 569275732Sjmg break; 570275732Sjmg case CRYPTO_AES_XTS: 571275732Sjmg ivlen = 8; 572275732Sjmg break; 573275732Sjmg case CRYPTO_AES_NIST_GCM_16: 574275732Sjmg ivlen = 12; /* should support arbitarily larger */ 575275732Sjmg break; 576275732Sjmg } 577275732Sjmg 578285216Sjmg /* Setup iv */ 579285216Sjmg if (encflag) { 580285216Sjmg if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 581285216Sjmg bcopy(enccrd->crd_iv, iv, ivlen); 582285216Sjmg else 583285216Sjmg arc4rand(iv, ivlen, 0); 584285216Sjmg 585285216Sjmg if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) 586285216Sjmg crypto_copyback(crp->crp_flags, crp->crp_buf, 587285216Sjmg enccrd->crd_inject, ivlen, iv); 588285216Sjmg } else { 589285216Sjmg if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 590285216Sjmg bcopy(enccrd->crd_iv, iv, ivlen); 591285216Sjmg else 592285216Sjmg crypto_copydata(crp->crp_flags, crp->crp_buf, 593285216Sjmg enccrd->crd_inject, ivlen, iv); 594285216Sjmg } 595275732Sjmg 596275732Sjmg if (authcrd != NULL && !encflag) 597275732Sjmg crypto_copydata(crp->crp_flags, crp->crp_buf, 598275732Sjmg authcrd->crd_inject, GMAC_DIGEST_LEN, tag); 599275732Sjmg else 600275732Sjmg bzero(tag, sizeof tag); 601275732Sjmg 602275732Sjmg /* Do work */ 603275732Sjmg switch (ses->algo) { 604275732Sjmg case CRYPTO_AES_CBC: 605275732Sjmg if (encflag) 606267815Skib aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, 607285216Sjmg enccrd->crd_len, buf, buf, iv); 608275732Sjmg else 609275732Sjmg aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, 610285216Sjmg enccrd->crd_len, buf, iv); 611275732Sjmg break; 612275732Sjmg case CRYPTO_AES_ICM: 613275732Sjmg /* encryption & decryption are the same */ 614275732Sjmg aesni_encrypt_icm(ses->rounds, ses->enc_schedule, 615285216Sjmg enccrd->crd_len, buf, buf, iv); 616275732Sjmg break; 617275732Sjmg case CRYPTO_AES_XTS: 618275732Sjmg if (encflag) 619267815Skib aesni_encrypt_xts(ses->rounds, ses->enc_schedule, 620267815Skib ses->xts_schedule, enccrd->crd_len, buf, buf, 621285216Sjmg iv); 622267815Skib else 623267815Skib aesni_decrypt_xts(ses->rounds, ses->dec_schedule, 624267815Skib ses->xts_schedule, enccrd->crd_len, buf, buf, 625285216Sjmg iv); 626275732Sjmg break; 627275732Sjmg case CRYPTO_AES_NIST_GCM_16: 628275732Sjmg if (encflag) 629285216Sjmg AES_GCM_encrypt(buf, buf, authbuf, iv, tag, 630275732Sjmg enccrd->crd_len, authcrd->crd_len, ivlen, 631275732Sjmg ses->enc_schedule, ses->rounds); 632275732Sjmg else { 633285216Sjmg if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag, 634275732Sjmg enccrd->crd_len, authcrd->crd_len, ivlen, 635275732Sjmg ses->enc_schedule, ses->rounds)) 636275732Sjmg error = EBADMSG; 637267815Skib } 638275732Sjmg break; 639267815Skib } 640275732Sjmg 641267815Skib if (allocated) 642267815Skib crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 643267815Skib enccrd->crd_len, buf); 644275732Sjmg 645275732Sjmg if (!error && authcrd != NULL) { 646275732Sjmg crypto_copyback(crp->crp_flags, crp->crp_buf, 647275732Sjmg authcrd->crd_inject, GMAC_DIGEST_LEN, tag); 648275732Sjmg } 649275732Sjmg 650267815Skibout: 651285289Sjmg if (!kt) { 652285289Sjmg fpu_kern_leave(curthread, ctx); 653285289Sjmgout2: 654285289Sjmg RELEASE_CTX(ctxidx, ctx); 655285289Sjmg } 656285289Sjmg 657267815Skibout1: 658267815Skib if (allocated) { 659267815Skib bzero(buf, enccrd->crd_len); 660267815Skib free(buf, M_AESNI); 661267815Skib } 662275732Sjmg if (authallocated) 663275732Sjmg free(authbuf, M_AESNI); 664267815Skib return (error); 665267815Skib} 666