1149211Spjd/*- 2181477Spjd * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3149211Spjd * All rights reserved. 4149211Spjd * 5149211Spjd * Redistribution and use in source and binary forms, with or without 6149211Spjd * modification, are permitted provided that the following conditions 7149211Spjd * are met: 8149211Spjd * 1. Redistributions of source code must retain the above copyright 9149211Spjd * notice, this list of conditions and the following disclaimer. 10149211Spjd * 2. Redistributions in binary form must reproduce the above copyright 11149211Spjd * notice, this list of conditions and the following disclaimer in the 12149211Spjd * documentation and/or other materials provided with the distribution. 13160674Spjd * 14149211Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15149211Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16149211Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17149211Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18149211Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19149211Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20149211Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21149211Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22149211Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23149211Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24149211Spjd * SUCH DAMAGE. 25149211Spjd */ 26149211Spjd 27149211Spjd#include <sys/cdefs.h> 28149211Spjd__FBSDID("$FreeBSD$"); 29149211Spjd 30149211Spjd#include <sys/param.h> 31149211Spjd#include <sys/systm.h> 32149211Spjd#include <sys/kernel.h> 33149211Spjd#include <sys/module.h> 34149211Spjd#include <sys/lock.h> 35180626Spjd#include <sys/rwlock.h> 36149211Spjd#include <sys/malloc.h> 37149211Spjd#include <sys/libkern.h> 38187112Sjkim#if defined(__amd64__) || (defined(__i386__) && !defined(PC98)) 39149211Spjd#include <machine/cpufunc.h> 40149211Spjd#include <machine/cputypes.h> 41160325Smr#include <machine/md_var.h> 42160325Smr#include <machine/specialreg.h> 43149211Spjd#endif 44149211Spjd 45149211Spjd#include <opencrypto/cryptodev.h> 46149211Spjd 47160582Spjd#include <crypto/via/padlock.h> 48149211Spjd 49167755Ssam#include <sys/kobj.h> 50167755Ssam#include <sys/bus.h> 51167755Ssam#include "cryptodev_if.h" 52167755Ssam 53160582Spjd/* 54160582Spjd * Technical documentation about the PadLock engine can be found here: 55160582Spjd * 56160582Spjd * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf 57160582Spjd */ 58149211Spjd 59149211Spjdstruct padlock_softc { 60149211Spjd int32_t sc_cid; 61149211Spjd uint32_t sc_sid; 62181473Spjd TAILQ_HEAD(padlock_sessions_head, padlock_session) sc_sessions; 63180626Spjd struct rwlock sc_sessions_lock; 64149211Spjd}; 65149211Spjd 66167755Ssamstatic int padlock_newsession(device_t, uint32_t *sidp, struct cryptoini *cri); 67167755Ssamstatic int padlock_freesession(device_t, uint64_t tid); 68181477Spjdstatic void padlock_freesession_one(struct padlock_softc *sc, 69181477Spjd struct padlock_session *ses, int locked); 70167755Ssamstatic int padlock_process(device_t, struct cryptop *crp, int hint __unused); 71149211Spjd 72160582SpjdMALLOC_DEFINE(M_PADLOCK, "padlock_data", "PadLock Data"); 73149211Spjd 74167755Ssamstatic void 75188171Simppadlock_identify(driver_t *drv, device_t parent) 76167755Ssam{ 77167755Ssam /* NB: order 10 is so we get attached after h/w devices */ 78167755Ssam if (device_find_child(parent, "padlock", -1) == NULL && 79167755Ssam BUS_ADD_CHILD(parent, 10, "padlock", -1) == 0) 80167755Ssam panic("padlock: could not attach"); 81167755Ssam} 82167755Ssam 83149211Spjdstatic int 84167755Ssampadlock_probe(device_t dev) 85149211Spjd{ 86160582Spjd char capp[256]; 87160582Spjd 88187112Sjkim#if defined(__amd64__) || (defined(__i386__) && !defined(PC98)) 89160582Spjd /* If there is no AES support, we has nothing to do here. */ 90160325Smr if (!(via_feature_xcrypt & VIA_HAS_AES)) { 91167755Ssam device_printf(dev, "No ACE support.\n"); 92149211Spjd return (EINVAL); 93160582Spjd } 94160582Spjd strlcpy(capp, "AES-CBC", sizeof(capp)); 95160582Spjd#if 0 96160582Spjd strlcat(capp, ",AES-EBC", sizeof(capp)); 97160582Spjd strlcat(capp, ",AES-CFB", sizeof(capp)); 98160582Spjd strlcat(capp, ",AES-OFB", sizeof(capp)); 99160582Spjd#endif 100160582Spjd if (via_feature_xcrypt & VIA_HAS_SHA) { 101160582Spjd strlcat(capp, ",SHA1", sizeof(capp)); 102160582Spjd strlcat(capp, ",SHA256", sizeof(capp)); 103160582Spjd } 104160582Spjd#if 0 105160582Spjd if (via_feature_xcrypt & VIA_HAS_AESCTR) 106160582Spjd strlcat(capp, ",AES-CTR", sizeof(capp)); 107160582Spjd if (via_feature_xcrypt & VIA_HAS_MM) 108160582Spjd strlcat(capp, ",RSA", sizeof(capp)); 109160582Spjd#endif 110167755Ssam device_set_desc_copy(dev, capp); 111167755Ssam return (0); 112149211Spjd#else 113149211Spjd return (EINVAL); 114149211Spjd#endif 115167755Ssam} 116149211Spjd 117167755Ssamstatic int 118167755Ssampadlock_attach(device_t dev) 119167755Ssam{ 120167755Ssam struct padlock_softc *sc = device_get_softc(dev); 121167755Ssam 122149211Spjd TAILQ_INIT(&sc->sc_sessions); 123149211Spjd sc->sc_sid = 1; 124149211Spjd 125167755Ssam sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 126149211Spjd if (sc->sc_cid < 0) { 127167755Ssam device_printf(dev, "Could not get crypto driver id.\n"); 128149211Spjd return (ENOMEM); 129160785Spjd } 130149211Spjd 131180626Spjd rw_init(&sc->sc_sessions_lock, "padlock_lock"); 132167755Ssam crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 133167755Ssam crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 134167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 135167755Ssam crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0); 136167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 137167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 138167755Ssam crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 139149211Spjd return (0); 140149211Spjd} 141149211Spjd 142149211Spjdstatic int 143167755Ssampadlock_detach(device_t dev) 144149211Spjd{ 145167755Ssam struct padlock_softc *sc = device_get_softc(dev); 146149211Spjd struct padlock_session *ses; 147149211Spjd 148180626Spjd rw_wlock(&sc->sc_sessions_lock); 149149211Spjd TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 150167755Ssam if (ses->ses_used) { 151180626Spjd rw_wunlock(&sc->sc_sessions_lock); 152167755Ssam device_printf(dev, 153167755Ssam "Cannot detach, sessions still active.\n"); 154167755Ssam return (EBUSY); 155167755Ssam } 156149211Spjd } 157181476Spjd while ((ses = TAILQ_FIRST(&sc->sc_sessions)) != NULL) { 158149211Spjd TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 159231979Skib fpu_kern_free_ctx(ses->ses_fpu_ctx); 160160582Spjd free(ses, M_PADLOCK); 161149211Spjd } 162180626Spjd rw_destroy(&sc->sc_sessions_lock); 163149211Spjd crypto_unregister_all(sc->sc_cid); 164149211Spjd return (0); 165149211Spjd} 166149211Spjd 167149211Spjdstatic int 168167755Ssampadlock_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) 169149211Spjd{ 170167755Ssam struct padlock_softc *sc = device_get_softc(dev); 171149211Spjd struct padlock_session *ses = NULL; 172159279Spjd struct cryptoini *encini, *macini; 173208834Skib struct thread *td; 174215864Skib int error, saved_ctx; 175149211Spjd 176167755Ssam if (sidp == NULL || cri == NULL) 177149211Spjd return (EINVAL); 178159279Spjd 179159279Spjd encini = macini = NULL; 180159279Spjd for (; cri != NULL; cri = cri->cri_next) { 181159279Spjd switch (cri->cri_alg) { 182159279Spjd case CRYPTO_NULL_HMAC: 183159279Spjd case CRYPTO_MD5_HMAC: 184159279Spjd case CRYPTO_SHA1_HMAC: 185159279Spjd case CRYPTO_RIPEMD160_HMAC: 186159279Spjd case CRYPTO_SHA2_256_HMAC: 187159279Spjd case CRYPTO_SHA2_384_HMAC: 188159279Spjd case CRYPTO_SHA2_512_HMAC: 189159279Spjd if (macini != NULL) 190159279Spjd return (EINVAL); 191159279Spjd macini = cri; 192159279Spjd break; 193159279Spjd case CRYPTO_AES_CBC: 194159279Spjd if (encini != NULL) 195159279Spjd return (EINVAL); 196159279Spjd encini = cri; 197159279Spjd break; 198159279Spjd default: 199159279Spjd return (EINVAL); 200159279Spjd } 201149211Spjd } 202159279Spjd 203159279Spjd /* 204159279Spjd * We only support HMAC algorithms to be able to work with 205171167Sgnn * ipsec(4), so if we are asked only for authentication without 206159279Spjd * encryption, don't pretend we can accellerate it. 207159279Spjd */ 208159279Spjd if (encini == NULL) 209149211Spjd return (EINVAL); 210149211Spjd 211149211Spjd /* 212149211Spjd * Let's look for a free session structure. 213149211Spjd */ 214180626Spjd rw_wlock(&sc->sc_sessions_lock); 215149211Spjd /* 216149211Spjd * Free sessions goes first, so if first session is used, we need to 217149211Spjd * allocate one. 218149211Spjd */ 219149211Spjd ses = TAILQ_FIRST(&sc->sc_sessions); 220181478Spjd if (ses == NULL || ses->ses_used) { 221160582Spjd ses = malloc(sizeof(*ses), M_PADLOCK, M_NOWAIT | M_ZERO); 222181475Spjd if (ses == NULL) { 223181475Spjd rw_wunlock(&sc->sc_sessions_lock); 224149211Spjd return (ENOMEM); 225181475Spjd } 226231979Skib ses->ses_fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL | 227231979Skib FPU_KERN_NOWAIT); 228231979Skib if (ses->ses_fpu_ctx == NULL) { 229231979Skib free(ses, M_PADLOCK); 230231979Skib rw_wunlock(&sc->sc_sessions_lock); 231231979Skib return (ENOMEM); 232231979Skib } 233149211Spjd ses->ses_id = sc->sc_sid++; 234181478Spjd } else { 235181478Spjd TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 236149211Spjd } 237181478Spjd ses->ses_used = 1; 238181478Spjd TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 239181475Spjd rw_wunlock(&sc->sc_sessions_lock); 240149211Spjd 241160582Spjd error = padlock_cipher_setup(ses, encini); 242160582Spjd if (error != 0) { 243181477Spjd padlock_freesession_one(sc, ses, 0); 244160582Spjd return (error); 245149211Spjd } 246149211Spjd 247159279Spjd if (macini != NULL) { 248208834Skib td = curthread; 249215864Skib if (!is_fpu_kern_thread(0)) { 250231979Skib error = fpu_kern_enter(td, ses->ses_fpu_ctx, 251215864Skib FPU_KERN_NORMAL); 252215864Skib saved_ctx = 1; 253215864Skib } else { 254215864Skib error = 0; 255215864Skib saved_ctx = 0; 256215864Skib } 257208834Skib if (error == 0) { 258208834Skib error = padlock_hash_setup(ses, macini); 259215864Skib if (saved_ctx) 260231979Skib fpu_kern_leave(td, ses->ses_fpu_ctx); 261208834Skib } 262160582Spjd if (error != 0) { 263181477Spjd padlock_freesession_one(sc, ses, 0); 264160582Spjd return (error); 265159279Spjd } 266149211Spjd } 267149211Spjd 268149211Spjd *sidp = ses->ses_id; 269149211Spjd return (0); 270149211Spjd} 271149211Spjd 272181477Spjdstatic void 273181477Spjdpadlock_freesession_one(struct padlock_softc *sc, struct padlock_session *ses, 274181477Spjd int locked) 275181477Spjd{ 276231979Skib struct fpu_kern_ctx *ctx; 277181477Spjd uint32_t sid = ses->ses_id; 278181477Spjd 279181477Spjd if (!locked) 280181477Spjd rw_wlock(&sc->sc_sessions_lock); 281181477Spjd TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 282181477Spjd padlock_hash_free(ses); 283231979Skib ctx = ses->ses_fpu_ctx; 284181477Spjd bzero(ses, sizeof(*ses)); 285181477Spjd ses->ses_used = 0; 286181477Spjd ses->ses_id = sid; 287231979Skib ses->ses_fpu_ctx = ctx; 288181477Spjd TAILQ_INSERT_HEAD(&sc->sc_sessions, ses, ses_next); 289181477Spjd if (!locked) 290181477Spjd rw_wunlock(&sc->sc_sessions_lock); 291181477Spjd} 292181477Spjd 293149211Spjdstatic int 294167755Ssampadlock_freesession(device_t dev, uint64_t tid) 295149211Spjd{ 296167755Ssam struct padlock_softc *sc = device_get_softc(dev); 297149211Spjd struct padlock_session *ses; 298149211Spjd uint32_t sid = ((uint32_t)tid) & 0xffffffff; 299149211Spjd 300180626Spjd rw_wlock(&sc->sc_sessions_lock); 301181473Spjd TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, padlock_sessions_head, 302181473Spjd ses_next) { 303149211Spjd if (ses->ses_id == sid) 304149211Spjd break; 305149211Spjd } 306149211Spjd if (ses == NULL) { 307180626Spjd rw_wunlock(&sc->sc_sessions_lock); 308149211Spjd return (EINVAL); 309149211Spjd } 310181477Spjd padlock_freesession_one(sc, ses, 1); 311180626Spjd rw_wunlock(&sc->sc_sessions_lock); 312149211Spjd return (0); 313149211Spjd} 314149211Spjd 315149211Spjdstatic int 316167755Ssampadlock_process(device_t dev, struct cryptop *crp, int hint __unused) 317149211Spjd{ 318167755Ssam struct padlock_softc *sc = device_get_softc(dev); 319160582Spjd struct padlock_session *ses = NULL; 320159279Spjd struct cryptodesc *crd, *enccrd, *maccrd; 321159279Spjd int error = 0; 322149211Spjd 323159405Spjd enccrd = maccrd = NULL; 324159405Spjd 325185026Sphilip /* Sanity check. */ 326185026Sphilip if (crp == NULL) 327185026Sphilip return (EINVAL); 328185026Sphilip 329185026Sphilip if (crp->crp_callback == NULL || crp->crp_desc == NULL) { 330159279Spjd error = EINVAL; 331149211Spjd goto out; 332149211Spjd } 333159279Spjd 334159279Spjd for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { 335159279Spjd switch (crd->crd_alg) { 336159279Spjd case CRYPTO_NULL_HMAC: 337159279Spjd case CRYPTO_MD5_HMAC: 338159279Spjd case CRYPTO_SHA1_HMAC: 339159279Spjd case CRYPTO_RIPEMD160_HMAC: 340159279Spjd case CRYPTO_SHA2_256_HMAC: 341159279Spjd case CRYPTO_SHA2_384_HMAC: 342159279Spjd case CRYPTO_SHA2_512_HMAC: 343159279Spjd if (maccrd != NULL) { 344159279Spjd error = EINVAL; 345159279Spjd goto out; 346159279Spjd } 347159279Spjd maccrd = crd; 348159279Spjd break; 349159279Spjd case CRYPTO_AES_CBC: 350159279Spjd if (enccrd != NULL) { 351159279Spjd error = EINVAL; 352159279Spjd goto out; 353159279Spjd } 354159279Spjd enccrd = crd; 355159279Spjd break; 356159279Spjd default: 357159279Spjd return (EINVAL); 358159279Spjd } 359149211Spjd } 360159279Spjd if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { 361159279Spjd error = EINVAL; 362157899Spjd goto out; 363157899Spjd } 364149211Spjd 365180626Spjd rw_rlock(&sc->sc_sessions_lock); 366181473Spjd TAILQ_FOREACH_REVERSE(ses, &sc->sc_sessions, padlock_sessions_head, 367181473Spjd ses_next) { 368149211Spjd if (ses->ses_id == (crp->crp_sid & 0xffffffff)) 369149211Spjd break; 370149211Spjd } 371180626Spjd rw_runlock(&sc->sc_sessions_lock); 372149211Spjd if (ses == NULL) { 373159279Spjd error = EINVAL; 374149211Spjd goto out; 375149211Spjd } 376149211Spjd 377159279Spjd /* Perform data authentication if requested before encryption. */ 378159279Spjd if (maccrd != NULL && maccrd->crd_next == enccrd) { 379160582Spjd error = padlock_hash_process(ses, maccrd, crp); 380159279Spjd if (error != 0) 381159279Spjd goto out; 382149211Spjd } 383149211Spjd 384160582Spjd error = padlock_cipher_process(ses, enccrd, crp); 385160582Spjd if (error != 0) 386160582Spjd goto out; 387149211Spjd 388159279Spjd /* Perform data authentication if requested after encryption. */ 389159279Spjd if (maccrd != NULL && enccrd->crd_next == maccrd) { 390160582Spjd error = padlock_hash_process(ses, maccrd, crp); 391159279Spjd if (error != 0) 392159279Spjd goto out; 393149211Spjd } 394149211Spjd 395149211Spjdout: 396160582Spjd#if 0 397160582Spjd /* 398160582Spjd * This code is not necessary, because contexts will be freed on next 399160582Spjd * padlock_setup_mackey() call or at padlock_freesession() call. 400160582Spjd */ 401160582Spjd if (ses != NULL && maccrd != NULL && 402160582Spjd (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 403160582Spjd padlock_free_ctx(ses->ses_axf, ses->ses_ictx); 404160582Spjd padlock_free_ctx(ses->ses_axf, ses->ses_octx); 405149211Spjd } 406160582Spjd#endif 407159279Spjd crp->crp_etype = error; 408149211Spjd crypto_done(crp); 409159279Spjd return (error); 410149211Spjd} 411149211Spjd 412167755Ssamstatic device_method_t padlock_methods[] = { 413167755Ssam DEVMETHOD(device_identify, padlock_identify), 414167755Ssam DEVMETHOD(device_probe, padlock_probe), 415167755Ssam DEVMETHOD(device_attach, padlock_attach), 416167755Ssam DEVMETHOD(device_detach, padlock_detach), 417149211Spjd 418167755Ssam DEVMETHOD(cryptodev_newsession, padlock_newsession), 419167755Ssam DEVMETHOD(cryptodev_freesession,padlock_freesession), 420167755Ssam DEVMETHOD(cryptodev_process, padlock_process), 421149211Spjd 422167755Ssam {0, 0}, 423167755Ssam}; 424167755Ssam 425167755Ssamstatic driver_t padlock_driver = { 426149211Spjd "padlock", 427167755Ssam padlock_methods, 428167755Ssam sizeof(struct padlock_softc), 429149211Spjd}; 430167755Ssamstatic devclass_t padlock_devclass; 431167755Ssam 432167755Ssam/* XXX where to attach */ 433167755SsamDRIVER_MODULE(padlock, nexus, padlock_driver, padlock_devclass, 0, 0); 434149211SpjdMODULE_VERSION(padlock, 1); 435149211SpjdMODULE_DEPEND(padlock, crypto, 1, 1, 1); 436