1149211Spjd/*- 2181477Spjd * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3149211Spjd * All rights reserved. 4149211Spjd * 5149211Spjd * Redistribution and use in source and binary forms, with or without 6149211Spjd * modification, are permitted provided that the following conditions 7149211Spjd * are met: 8149211Spjd * 1. Redistributions of source code must retain the above copyright 9149211Spjd * notice, this list of conditions and the following disclaimer. 10149211Spjd * 2. Redistributions in binary form must reproduce the above copyright 11149211Spjd * notice, this list of conditions and the following disclaimer in the 12149211Spjd * documentation and/or other materials provided with the distribution. 13160674Spjd * 14149211Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15149211Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16149211Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17149211Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18149211Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19149211Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20149211Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21149211Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22149211Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23149211Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24149211Spjd * SUCH DAMAGE. 25149211Spjd */ 26149211Spjd 27149211Spjd#include <sys/cdefs.h> 28149211Spjd__FBSDID("$FreeBSD: releng/10.3/sys/crypto/via/padlock.c 268033 2014-06-30 09:48:44Z kib $"); 29149211Spjd 30149211Spjd#include <sys/param.h> 31149211Spjd#include <sys/systm.h> 32149211Spjd#include <sys/kernel.h> 33149211Spjd#include <sys/module.h> 34149211Spjd#include <sys/lock.h> 35180626Spjd#include <sys/rwlock.h> 36149211Spjd#include <sys/malloc.h> 37149211Spjd#include <sys/libkern.h> 38187112Sjkim#if defined(__amd64__) || (defined(__i386__) && !defined(PC98)) 39149211Spjd#include <machine/cpufunc.h> 40149211Spjd#include <machine/cputypes.h> 41160325Smr#include <machine/md_var.h> 42160325Smr#include <machine/specialreg.h> 43149211Spjd#endif 44149211Spjd 45149211Spjd#include <opencrypto/cryptodev.h> 46149211Spjd 47160582Spjd#include <crypto/via/padlock.h> 48149211Spjd 49167755Ssam#include <sys/kobj.h> 50167755Ssam#include <sys/bus.h> 51167755Ssam#include "cryptodev_if.h" 52167755Ssam 53160582Spjd/* 54160582Spjd * Technical documentation about the PadLock engine can be found here: 55160582Spjd * 56160582Spjd * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf 57160582Spjd */ 58149211Spjd 59149211Spjdstruct padlock_softc { 60149211Spjd int32_t sc_cid; 61149211Spjd uint32_t sc_sid; 62181473Spjd TAILQ_HEAD(padlock_sessions_head, padlock_session) sc_sessions; 63180626Spjd struct rwlock sc_sessions_lock; 64149211Spjd}; 65149211Spjd 66167755Ssamstatic int padlock_newsession(device_t, uint32_t *sidp, struct cryptoini *cri); 67167755Ssamstatic int padlock_freesession(device_t, uint64_t tid); 68181477Spjdstatic void padlock_freesession_one(struct padlock_softc *sc, 69181477Spjd struct padlock_session *ses, int locked); 70167755Ssamstatic int padlock_process(device_t, struct cryptop *crp, int hint __unused); 71149211Spjd 72160582SpjdMALLOC_DEFINE(M_PADLOCK, "padlock_data", "PadLock Data"); 73149211Spjd 74167755Ssamstatic void 75188171Simppadlock_identify(driver_t *drv, device_t parent) 76167755Ssam{ 77167755Ssam /* NB: order 10 is so we get attached after h/w devices */ 78167755Ssam if (device_find_child(parent, "padlock", -1) == NULL && 79167755Ssam BUS_ADD_CHILD(parent, 10, "padlock", -1) == 0) 80167755Ssam panic("padlock: could not attach"); 81167755Ssam} 82167755Ssam 83149211Spjdstatic int 84167755Ssampadlock_probe(device_t dev) 85149211Spjd{ 86160582Spjd char capp[256]; 87160582Spjd 88187112Sjkim#if defined(__amd64__) || (defined(__i386__) && !defined(PC98)) 89160582Spjd /* If there is no AES support, we has nothing to do here. */ 90160325Smr if (!(via_feature_xcrypt & VIA_HAS_AES)) { 91167755Ssam device_printf(dev, "No ACE support.\n"); 92149211Spjd return (EINVAL); 93160582Spjd } 94160582Spjd strlcpy(capp, "AES-CBC", sizeof(capp)); 95160582Spjd#if 0 96160582Spjd strlcat(capp, ",AES-EBC", sizeof(capp)); 97160582Spjd strlcat(capp, ",AES-CFB", sizeof(capp)); 98160582Spjd strlcat(capp, ",AES-OFB", sizeof(capp)); 99160582Spjd#endif 100160582Spjd if (via_feature_xcrypt & VIA_HAS_SHA) { 101160582Spjd strlcat(capp, ",SHA1", sizeof(capp)); 102160582Spjd strlcat(capp, ",SHA256", sizeof(capp)); 103160582Spjd } 104160582Spjd#if 0 105160582Spjd if (via_feature_xcrypt & VIA_HAS_AESCTR) 106160582Spjd strlcat(capp, ",AES-CTR", sizeof(capp)); 107160582Spjd if (via_feature_xcrypt & VIA_HAS_MM) 108160582Spjd strlcat(capp, ",RSA", sizeof(capp)); 109160582Spjd#endif 110167755Ssam device_set_desc_copy(dev, capp); 111167755Ssam return (0); 112149211Spjd#else 113149211Spjd return (EINVAL); 114149211Spjd#endif 115167755Ssam} 116149211Spjd 117167755Ssamstatic int 118167755Ssampadlock_attach(device_t dev) 119167755Ssam{ 120167755Ssam struct padlock_softc *sc = device_get_softc(dev); 121167755Ssam 122149211Spjd TAILQ_INIT(&sc->sc_sessions); 123149211Spjd sc->sc_sid = 1; 124149211Spjd 125167755Ssam sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 126149211Spjd if (sc->sc_cid < 0) { 127167755Ssam device_printf(dev, "Could not get crypto driver id.\n"); 128149211Spjd return (ENOMEM); 129160785Spjd } 130149211Spjd 131180626Spjd rw_init(&sc->sc_sessions_lock, "padlock_lock"); 132167755Ssam crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 133167755Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 134167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 135167755Ssam crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0); 136167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 137167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 138167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 139149211Spjd return (0); 140149211Spjd} 141149211Spjd 142149211Spjdstatic int 143167755Ssampadlock_detach(device_t dev) 144149211Spjd{ 145167755Ssam struct padlock_softc *sc = device_get_softc(dev); 146149211Spjd struct padlock_session *ses; 147149211Spjd 148180626Spjd rw_wlock(&sc->sc_sessions_lock); 149149211Spjd TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 150167755Ssam if (ses->ses_used) { 151180626Spjd rw_wunlock(&sc->sc_sessions_lock); 152167755Ssam device_printf(dev, 153167755Ssam "Cannot detach, sessions still active.\n"); 154167755Ssam return (EBUSY); 155167755Ssam } 156149211Spjd } 157181476Spjd while ((ses = TAILQ_FIRST(&sc->sc_sessions)) != NULL) { 158149211Spjd TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 159230426Skib fpu_kern_free_ctx(ses->ses_fpu_ctx); 160160582Spjd free(ses, M_PADLOCK); 161149211Spjd } 162180626Spjd rw_destroy(&sc->sc_sessions_lock); 163149211Spjd crypto_unregister_all(sc->sc_cid); 164149211Spjd return (0); 165149211Spjd} 166149211Spjd 167149211Spjdstatic int 168167755Ssampadlock_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 169149211Spjd{ 170167755Ssam struct padlock_softc *sc = device_get_softc(dev); 171149211Spjd struct padlock_session *ses = NULL; 172159279Spjd struct cryptoini *encini, *macini; 173208834Skib struct thread *td; 174268033Skib int error; 175149211Spjd 176167755Ssam if (sidp == NULL || cri == NULL) 177149211Spjd return (EINVAL); 178159279Spjd 179159279Spjd encini = macini = NULL; 180159279Spjd for (; cri != NULL; cri = cri->cri_next) { 181159279Spjd switch (cri->cri_alg) { 182159279Spjd case CRYPTO_NULL_HMAC: 183159279Spjd case CRYPTO_MD5_HMAC: 184159279Spjd case CRYPTO_SHA1_HMAC: 185159279Spjd case CRYPTO_RIPEMD160_HMAC: 186159279Spjd case CRYPTO_SHA2_256_HMAC: 187159279Spjd case CRYPTO_SHA2_384_HMAC: 188159279Spjd case CRYPTO_SHA2_512_HMAC: 189159279Spjd if (macini != NULL) 190159279Spjd return (EINVAL); 191159279Spjd macini = cri; 192159279Spjd break; 193159279Spjd case CRYPTO_AES_CBC: 194159279Spjd if (encini != NULL) 195159279Spjd return (EINVAL); 196159279Spjd encini = cri; 197159279Spjd break; 198159279Spjd default: 199159279Spjd return (EINVAL); 200159279Spjd } 201149211Spjd } 202159279Spjd 203159279Spjd /* 204159279Spjd * We only support HMAC algorithms to be able to work with 205171167Sgnn * ipsec(4), so if we are asked only for authentication without 206159279Spjd * encryption, don't pretend we can accellerate it. 207159279Spjd */ 208159279Spjd if (encini == NULL) 209149211Spjd return (EINVAL); 210149211Spjd 211149211Spjd /* 212149211Spjd * Let's look for a free session structure. 213149211Spjd */ 214180626Spjd rw_wlock(&sc->sc_sessions_lock); 215149211Spjd /* 216149211Spjd * Free sessions goes first, so if first session is used, we need to 217149211Spjd * allocate one. 218149211Spjd */ 219149211Spjd ses = TAILQ_FIRST(&sc->sc_sessions); 220181478Spjd if (ses == NULL || ses->ses_used) { 221160582Spjd ses = malloc(sizeof(*ses), M_PADLOCK, M_NOWAIT | M_ZERO); 222181475Spjd if (ses == NULL) { 223181475Spjd rw_wunlock(&sc->sc_sessions_lock); 224149211Spjd return (ENOMEM); 225181475Spjd } 226230426Skib ses->ses_fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL | 227230426Skib FPU_KERN_NOWAIT); 228230426Skib if (ses->ses_fpu_ctx == NULL) { 229230426Skib free(ses, M_PADLOCK); 230230426Skib rw_wunlock(&sc->sc_sessions_lock); 231230426Skib return (ENOMEM); 232230426Skib } 233149211Spjd ses->ses_id = sc->sc_sid++; 234181478Spjd } else { 235181478Spjd TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 236149211Spjd } 237181478Spjd ses->ses_used = 1; 238181478Spjd TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 239181475Spjd rw_wunlock(&sc->sc_sessions_lock); 240149211Spjd 241160582Spjd error = padlock_cipher_setup(ses, encini); 242160582Spjd if (error != 0) { 243181477Spjd padlock_freesession_one(sc, ses, 0); 244160582Spjd return (error); 245149211Spjd } 246149211Spjd 247159279Spjd if (macini != NULL) { 248208834Skib td = curthread; 249268033Skib error = fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | 250268033Skib FPU_KERN_KTHR); 251208834Skib if (error == 0) { 252208834Skib error = padlock_hash_setup(ses, macini); 253268033Skib fpu_kern_leave(td, ses->ses_fpu_ctx); 254208834Skib } 255160582Spjd if (error != 0) { 256181477Spjd padlock_freesession_one(sc, ses, 0); 257160582Spjd return (error); 258159279Spjd } 259149211Spjd } 260149211Spjd 261149211Spjd *sidp = ses->ses_id; 262149211Spjd return (0); 263149211Spjd} 264149211Spjd 265181477Spjdstatic void 266181477Spjdpadlock_freesession_one(struct padlock_softc *sc, struct padlock_session *ses, 267181477Spjd int locked) 268181477Spjd{ 269230426Skib struct fpu_kern_ctx *ctx; 270181477Spjd uint32_t sid = ses->ses_id; 271181477Spjd 272181477Spjd if (!locked) 273181477Spjd rw_wlock(&sc->sc_sessions_lock); 274181477Spjd TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 275181477Spjd padlock_hash_free(ses); 276230426Skib ctx = ses->ses_fpu_ctx; 277181477Spjd bzero(ses, sizeof(*ses)); 278181477Spjd ses->ses_used = 0; 279181477Spjd ses->ses_id = sid; 280230426Skib ses->ses_fpu_ctx = ctx; 281181477Spjd TAILQ_INSERT_HEAD(&sc->sc_sessions, ses, ses_next); 282181477Spjd if (!locked) 283181477Spjd rw_wunlock(&sc->sc_sessions_lock); 284181477Spjd} 285181477Spjd 286149211Spjdstatic int 287167755Ssampadlock_freesession(device_t dev, uint64_t tid) 288149211Spjd{ 289167755Ssam struct padlock_softc *sc = device_get_softc(dev); 290149211Spjd struct padlock_session *ses; 291149211Spjd uint32_t sid = ((uint32_t)tid) & 0xffffffff; 292149211Spjd 293180626Spjd rw_wlock(&sc->sc_sessions_lock); 294181473Spjd TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, padlock_sessions_head, 295181473Spjd ses_next) { 296149211Spjd if (ses->ses_id == sid) 297149211Spjd break; 298149211Spjd } 299149211Spjd if (ses == NULL) { 300180626Spjd rw_wunlock(&sc->sc_sessions_lock); 301149211Spjd return (EINVAL); 302149211Spjd } 303181477Spjd padlock_freesession_one(sc, ses, 1); 304180626Spjd rw_wunlock(&sc->sc_sessions_lock); 305149211Spjd return (0); 306149211Spjd} 307149211Spjd 308149211Spjdstatic int 309167755Ssampadlock_process(device_t dev, struct cryptop *crp, int hint __unused) 310149211Spjd{ 311167755Ssam struct padlock_softc *sc = device_get_softc(dev); 312160582Spjd struct padlock_session *ses = NULL; 313159279Spjd struct cryptodesc *crd, *enccrd, *maccrd; 314159279Spjd int error = 0; 315149211Spjd 316159405Spjd enccrd = maccrd = NULL; 317159405Spjd 318185026Sphilip /* Sanity check. */ 319185026Sphilip if (crp == NULL) 320185026Sphilip return (EINVAL); 321185026Sphilip 322185026Sphilip if (crp->crp_callback == NULL || crp->crp_desc == NULL) { 323159279Spjd error = EINVAL; 324149211Spjd goto out; 325149211Spjd } 326159279Spjd 327159279Spjd for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { 328159279Spjd switch (crd->crd_alg) { 329159279Spjd case CRYPTO_NULL_HMAC: 330159279Spjd case CRYPTO_MD5_HMAC: 331159279Spjd case CRYPTO_SHA1_HMAC: 332159279Spjd case CRYPTO_RIPEMD160_HMAC: 333159279Spjd case CRYPTO_SHA2_256_HMAC: 334159279Spjd case CRYPTO_SHA2_384_HMAC: 335159279Spjd case CRYPTO_SHA2_512_HMAC: 336159279Spjd if (maccrd != NULL) { 337159279Spjd error = EINVAL; 338159279Spjd goto out; 339159279Spjd } 340159279Spjd maccrd = crd; 341159279Spjd break; 342159279Spjd case CRYPTO_AES_CBC: 343159279Spjd if (enccrd != NULL) { 344159279Spjd error = EINVAL; 345159279Spjd goto out; 346159279Spjd } 347159279Spjd enccrd = crd; 348159279Spjd break; 349159279Spjd default: 350159279Spjd return (EINVAL); 351159279Spjd } 352149211Spjd } 353159279Spjd if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { 354159279Spjd error = EINVAL; 355157899Spjd goto out; 356157899Spjd } 357149211Spjd 358180626Spjd rw_rlock(&sc->sc_sessions_lock); 359181473Spjd TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, padlock_sessions_head, 360181473Spjd ses_next) { 361149211Spjd if (ses->ses_id == (crp->crp_sid & 0xffffffff)) 362149211Spjd break; 363149211Spjd } 364180626Spjd rw_runlock(&sc->sc_sessions_lock); 365149211Spjd if (ses == NULL) { 366159279Spjd error = EINVAL; 367149211Spjd goto out; 368149211Spjd } 369149211Spjd 370159279Spjd /* Perform data authentication if requested before encryption. */ 371159279Spjd if (maccrd != NULL && maccrd->crd_next == enccrd) { 372160582Spjd error = padlock_hash_process(ses, maccrd, crp); 373159279Spjd if (error != 0) 374159279Spjd goto out; 375149211Spjd } 376149211Spjd 377160582Spjd error = padlock_cipher_process(ses, enccrd, crp); 378160582Spjd if (error != 0) 379160582Spjd goto out; 380149211Spjd 381159279Spjd /* Perform data authentication if requested after encryption. */ 382159279Spjd if (maccrd != NULL && enccrd->crd_next == maccrd) { 383160582Spjd error = padlock_hash_process(ses, maccrd, crp); 384159279Spjd if (error != 0) 385159279Spjd goto out; 386149211Spjd } 387149211Spjd 388149211Spjdout: 389160582Spjd#if 0 390160582Spjd /* 391160582Spjd * This code is not necessary, because contexts will be freed on next 392160582Spjd * padlock_setup_mackey() call or at padlock_freesession() call. 393160582Spjd */ 394160582Spjd if (ses != NULL && maccrd != NULL && 395160582Spjd (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 396160582Spjd padlock_free_ctx(ses->ses_axf, ses->ses_ictx); 397160582Spjd padlock_free_ctx(ses->ses_axf, ses->ses_octx); 398149211Spjd } 399160582Spjd#endif 400159279Spjd crp->crp_etype = error; 401149211Spjd crypto_done(crp); 402159279Spjd return (error); 403149211Spjd} 404149211Spjd 405167755Ssamstatic device_method_t padlock_methods[] = { 406167755Ssam DEVMETHOD(device_identify, padlock_identify), 407167755Ssam DEVMETHOD(device_probe, padlock_probe), 408167755Ssam DEVMETHOD(device_attach, padlock_attach), 409167755Ssam DEVMETHOD(device_detach, padlock_detach), 410149211Spjd 411167755Ssam DEVMETHOD(cryptodev_newsession, padlock_newsession), 412167755Ssam DEVMETHOD(cryptodev_freesession,padlock_freesession), 413167755Ssam DEVMETHOD(cryptodev_process, padlock_process), 414149211Spjd 415167755Ssam {0, 0}, 416167755Ssam}; 417167755Ssam 418167755Ssamstatic driver_t padlock_driver = { 419149211Spjd "padlock", 420167755Ssam padlock_methods, 421167755Ssam sizeof(struct padlock_softc), 422149211Spjd}; 423167755Ssamstatic devclass_t padlock_devclass; 424167755Ssam 425167755Ssam/* XXX where to attach */ 426167755SsamDRIVER_MODULE(padlock, nexus, padlock_driver, padlock_devclass, 0, 0); 427149211SpjdMODULE_VERSION(padlock, 1); 428149211SpjdMODULE_DEPEND(padlock, crypto, 1, 1, 1); 429