padlock.c revision 157899
1/*- 2 * Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2004 Mark R V Murray 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28/* $OpenBSD: via.c,v 1.3 2004/06/15 23:36:55 deraadt Exp $ */ 29/*- 30 * Copyright (c) 2003 Jason Wright 31 * Copyright (c) 2003, 2004 Theo de Raadt 32 * All rights reserved. 33 * 34 * Permission to use, copy, modify, and distribute this software for any 35 * purpose with or without fee is hereby granted, provided that the above 36 * copyright notice and this permission notice appear in all copies. 37 * 38 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 39 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 40 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 41 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 42 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 43 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 44 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 45 */ 46 47#include <sys/cdefs.h> 48__FBSDID("$FreeBSD: head/sys/crypto/via/padlock.c 157899 2006-04-20 06:31:44Z pjd $"); 49 50#include <sys/param.h> 51#include <sys/systm.h> 52#include <sys/kernel.h> 53#include <sys/module.h> 54#include <sys/lock.h> 55#include <sys/mutex.h> 56#include <sys/malloc.h> 57#include <sys/libkern.h> 58#include <sys/mbuf.h> 59#include <sys/uio.h> 60#if defined(__i386__) && !defined(PC98) 61#include <machine/cpufunc.h> 62#include <machine/cputypes.h> 63#endif 64 65#include <opencrypto/cryptodev.h> 66#include <crypto/rijndael/rijndael.h> 67 68 69#define PADLOCK_ROUND_COUNT_AES128 10 70#define PADLOCK_ROUND_COUNT_AES192 12 71#define PADLOCK_ROUND_COUNT_AES256 14 72 73#define PADLOCK_ALGORITHM_TYPE_AES 0 74 75#define PADLOCK_KEY_GENERATION_HW 0 76#define PADLOCK_KEY_GENERATION_SW 1 77 78#define PADLOCK_DIRECTION_ENCRYPT 0 79#define PADLOCK_DIRECTION_DECRYPT 1 80 81#define PADLOCK_KEY_SIZE_128 0 82#define PADLOCK_KEY_SIZE_192 1 83#define PADLOCK_KEY_SIZE_256 2 84 85union padlock_cw { 86 uint64_t raw; 87 struct { 88 u_int round_count : 4; 89 u_int algorithm_type : 3; 90 u_int key_generation : 1; 91 u_int intermediate : 1; 92 u_int direction : 1; 93 u_int key_size : 2; 94 u_int filler0 : 20; 95 u_int filler1 : 32; 96 u_int filler2 : 32; 97 u_int filler3 : 32; 98 } __field; 99}; 100#define cw_round_count __field.round_count 101#define cw_algorithm_type __field.algorithm_type 102#define cw_key_generation __field.key_generation 103#define cw_intermediate __field.intermediate 104#define cw_direction __field.direction 105#define cw_key_size __field.key_size 106#define cw_filler0 __field.filler0 107#define cw_filler1 __field.filler1 108#define cw_filler2 __field.filler2 109#define cw_filler3 __field.filler3 110 111struct padlock_session { 112 union padlock_cw ses_cw __aligned(16); 113 uint32_t ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */ 114 uint32_t ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */ 115 uint8_t ses_iv[16] __aligned(16); /* 128 bit aligned */ 116 int ses_used; 117 uint32_t ses_id; 118 TAILQ_ENTRY(padlock_session) ses_next; 119}; 120 121struct padlock_softc { 122 int32_t sc_cid; 123 uint32_t sc_sid; 124 TAILQ_HEAD(, padlock_session) sc_sessions; 125 struct mtx sc_sessions_mtx; 126}; 127 128static struct padlock_softc *padlock_sc; 129 130static int padlock_newsession(void *arg __unused, uint32_t *sidp, 131 struct cryptoini *cri); 132static int padlock_freesession(void *arg __unused, uint64_t tid); 133static int padlock_process(void *arg __unused, struct cryptop *crp, 134 int hint __unused); 135 136static __inline void 137padlock_cbc(void *in, void *out, size_t count, void *key, union padlock_cw *cw, 138 void *iv) 139{ 140#ifdef __GNUCLIKE_ASM 141 /* The .byte line is really VIA C3 "xcrypt-cbc" instruction */ 142 __asm __volatile( 143 "pushf \n\t" 144 "popf \n\t" 145 "rep \n\t" 146 ".byte 0x0f, 0xa7, 0xd0" 147 : "+a" (iv), "+c" (count), "+D" (out), "+S" (in) 148 : "b" (key), "d" (cw) 149 : "cc", "memory" 150 ); 151#endif 152} 153 154static int 155padlock_init(void) 156{ 157 struct padlock_softc *sc; 158#if defined(__i386__) && !defined(PC98) 159 u_int regs[4]; 160 int has_ace = 0; 161 162 if (cpu_class < CPUCLASS_586) 163 return (EINVAL); 164 do_cpuid(1, regs); 165 if ((regs[0] & 0xf) >= 3) { 166 do_cpuid(0xc0000000, regs); 167 if (regs[0] == 0xc0000001) { 168 do_cpuid(0xc0000001, regs); 169 if ((regs[3] & 0xc0) == 0xc0) 170 has_ace = 1; 171 } 172 } 173 if (!has_ace) { 174 printf("PADLOCK: No ACE support.\n"); 175 return (EINVAL); 176 } 177#else 178 return (EINVAL); 179#endif 180 181 padlock_sc = sc = malloc(sizeof(*padlock_sc), M_DEVBUF, 182 M_WAITOK | M_ZERO); 183 TAILQ_INIT(&sc->sc_sessions); 184 sc->sc_sid = 1; 185 186 sc->sc_cid = crypto_get_driverid(0); 187 if (sc->sc_cid < 0) { 188 printf("PADLOCK: Could not get crypto driver id.\n"); 189 free(padlock_sc, M_DEVBUF); 190 padlock_sc = NULL; 191 return (ENOMEM); 192 } 193 194 mtx_init(&sc->sc_sessions_mtx, "padlock_mtx", NULL, MTX_DEF); 195 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, padlock_newsession, 196 padlock_freesession, padlock_process, NULL); 197 return (0); 198} 199 200static int 201padlock_destroy(void) 202{ 203 struct padlock_softc *sc = padlock_sc; 204 struct padlock_session *ses; 205 u_int active = 0; 206 207 if (sc == NULL) 208 return (0); 209 mtx_lock(&sc->sc_sessions_mtx); 210 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 211 if (ses->ses_used) 212 active++; 213 } 214 if (active > 0) { 215 mtx_unlock(&sc->sc_sessions_mtx); 216 printf("PADLOCK: Cannot destroy, %u sessions active.\n", 217 active); 218 return (EBUSY); 219 } 220 padlock_sc = NULL; 221 for (ses = TAILQ_FIRST(&sc->sc_sessions); ses != NULL; 222 ses = TAILQ_FIRST(&sc->sc_sessions)) { 223 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 224 free(ses, M_DEVBUF); 225 } 226 mtx_destroy(&sc->sc_sessions_mtx); 227 crypto_unregister_all(sc->sc_cid); 228 free(sc, M_DEVBUF); 229 return (0); 230} 231 232static int 233padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri) 234{ 235 struct padlock_softc *sc = padlock_sc; 236 struct padlock_session *ses = NULL; 237 union padlock_cw *cw; 238 int i; 239 240 if (sc == NULL || sidp == NULL || cri == NULL || 241 cri->cri_next != NULL || cri->cri_alg != CRYPTO_AES_CBC) { 242 return (EINVAL); 243 } 244 if (cri->cri_klen != 128 && cri->cri_klen != 192 && 245 cri->cri_klen != 256) { 246 return (EINVAL); 247 } 248 249 /* 250 * Let's look for a free session structure. 251 */ 252 mtx_lock(&sc->sc_sessions_mtx); 253 /* 254 * Free sessions goes first, so if first session is used, we need to 255 * allocate one. 256 */ 257 ses = TAILQ_FIRST(&sc->sc_sessions); 258 if (ses == NULL || ses->ses_used) 259 ses = NULL; 260 else { 261 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 262 ses->ses_used = 1; 263 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 264 } 265 mtx_unlock(&sc->sc_sessions_mtx); 266 if (ses == NULL) { 267 ses = malloc(sizeof(*ses), M_DEVBUF, M_NOWAIT | M_ZERO); 268 if (ses == NULL) 269 return (ENOMEM); 270 ses->ses_used = 1; 271 mtx_lock(&sc->sc_sessions_mtx); 272 ses->ses_id = sc->sc_sid++; 273 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 274 mtx_unlock(&sc->sc_sessions_mtx); 275 } 276 277 cw = &ses->ses_cw; 278 bzero(cw, sizeof(*cw)); 279 cw->cw_algorithm_type = PADLOCK_ALGORITHM_TYPE_AES; 280 cw->cw_key_generation = PADLOCK_KEY_GENERATION_SW; 281 cw->cw_intermediate = 0; 282 switch (cri->cri_klen) { 283 case 128: 284 cw->cw_round_count = PADLOCK_ROUND_COUNT_AES128; 285 cw->cw_key_size = PADLOCK_KEY_SIZE_128; 286#ifdef HW_KEY_GENERATION 287 /* This doesn't buy us much, that's why it is commented out. */ 288 cw->cw_key_generation = PADLOCK_KEY_GENERATION_HW; 289#endif 290 break; 291 case 192: 292 cw->cw_round_count = PADLOCK_ROUND_COUNT_AES192; 293 cw->cw_key_size = PADLOCK_KEY_SIZE_192; 294 break; 295 case 256: 296 cw->cw_round_count = PADLOCK_ROUND_COUNT_AES256; 297 cw->cw_key_size = PADLOCK_KEY_SIZE_256; 298 break; 299 } 300 301 arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0); 302 303 if (cw->cw_key_generation == PADLOCK_KEY_GENERATION_SW) { 304 /* Build expanded keys for both directions */ 305 rijndaelKeySetupEnc(ses->ses_ekey, cri->cri_key, cri->cri_klen); 306 rijndaelKeySetupDec(ses->ses_dkey, cri->cri_key, cri->cri_klen); 307 for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) { 308 ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]); 309 ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]); 310 } 311 } else { 312 bcopy(cri->cri_key, ses->ses_ekey, cri->cri_klen); 313 bcopy(cri->cri_key, ses->ses_dkey, cri->cri_klen); 314 } 315 316 *sidp = ses->ses_id; 317 return (0); 318} 319 320static int 321padlock_freesession(void *arg __unused, uint64_t tid) 322{ 323 struct padlock_softc *sc = padlock_sc; 324 struct padlock_session *ses; 325 uint32_t sid = ((uint32_t)tid) & 0xffffffff; 326 327 if (sc == NULL) 328 return (EINVAL); 329 mtx_lock(&sc->sc_sessions_mtx); 330 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 331 if (ses->ses_id == sid) 332 break; 333 } 334 if (ses == NULL) { 335 mtx_unlock(&sc->sc_sessions_mtx); 336 return (EINVAL); 337 } 338 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 339 bzero(ses, sizeof(ses)); 340 ses->ses_used = 0; 341 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 342 mtx_unlock(&sc->sc_sessions_mtx); 343 return (0); 344} 345 346static int 347padlock_process(void *arg __unused, struct cryptop *crp, int hint __unused) 348{ 349 struct padlock_softc *sc = padlock_sc; 350 struct padlock_session *ses; 351 union padlock_cw *cw; 352 struct cryptodesc *crd = NULL; 353 uint32_t *key; 354 u_char *buf, *abuf; 355 int err = 0; 356 357 buf = NULL; 358 if (crp == NULL || crp->crp_callback == NULL) { 359 err = EINVAL; 360 goto out; 361 } 362 crd = crp->crp_desc; 363 if (crd == NULL || crd->crd_next != NULL || 364 crd->crd_alg != CRYPTO_AES_CBC || 365 (crd->crd_len % 16) != 0) { 366 err = EINVAL; 367 goto out; 368 } 369 if ((crd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 370 err = EINVAL; 371 goto out; 372 } 373 374 mtx_lock(&sc->sc_sessions_mtx); 375 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 376 if (ses->ses_id == (crp->crp_sid & 0xffffffff)) 377 break; 378 } 379 mtx_unlock(&sc->sc_sessions_mtx); 380 if (ses == NULL) { 381 err = EINVAL; 382 goto out; 383 } 384 385 buf = malloc(crd->crd_len + 16, M_DEVBUF, M_NOWAIT); 386 if (buf == NULL) { 387 err = ENOMEM; 388 goto out; 389 } 390 abuf = buf + 16 - ((uintptr_t)buf % 16); 391 392 cw = &ses->ses_cw; 393 cw->cw_filler0 = 0; 394 cw->cw_filler1 = 0; 395 cw->cw_filler2 = 0; 396 cw->cw_filler3 = 0; 397 if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) { 398 cw->cw_direction = PADLOCK_DIRECTION_ENCRYPT; 399 key = ses->ses_ekey; 400 if ((crd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 401 bcopy(crd->crd_iv, ses->ses_iv, 16); 402 403 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 404 if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) { 405 m_copyback((struct mbuf *)crp->crp_buf, 406 crd->crd_inject, 16, ses->ses_iv); 407 } else if ((crp->crp_flags & CRYPTO_F_IOV) != 0) { 408 cuio_copyback((struct uio *)crp->crp_buf, 409 crd->crd_inject, 16, ses->ses_iv); 410 } else { 411 bcopy(ses->ses_iv, 412 crp->crp_buf + crd->crd_inject, 16); 413 } 414 } 415 } else { 416 cw->cw_direction = PADLOCK_DIRECTION_DECRYPT; 417 key = ses->ses_dkey; 418 if ((crd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 419 bcopy(crd->crd_iv, ses->ses_iv, 16); 420 else { 421 if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) { 422 m_copydata((struct mbuf *)crp->crp_buf, 423 crd->crd_inject, 16, ses->ses_iv); 424 } else if ((crp->crp_flags & CRYPTO_F_IOV) != 0) { 425 cuio_copydata((struct uio *)crp->crp_buf, 426 crd->crd_inject, 16, ses->ses_iv); 427 } else { 428 bcopy(crp->crp_buf + crd->crd_inject, 429 ses->ses_iv, 16); 430 } 431 } 432 } 433 434 if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) { 435 m_copydata((struct mbuf *)crp->crp_buf, crd->crd_skip, 436 crd->crd_len, abuf); 437 } else if ((crp->crp_flags & CRYPTO_F_IOV) != 0) { 438 cuio_copydata((struct uio *)crp->crp_buf, crd->crd_skip, 439 crd->crd_len, abuf); 440 } else { 441 bcopy(crp->crp_buf + crd->crd_skip, abuf, crd->crd_len); 442 } 443 444 padlock_cbc(abuf, abuf, crd->crd_len / 16, key, cw, ses->ses_iv); 445 446 if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) { 447 m_copyback((struct mbuf *)crp->crp_buf, crd->crd_skip, 448 crd->crd_len, abuf); 449 } else if ((crp->crp_flags & CRYPTO_F_IOV) != 0) { 450 cuio_copyback((struct uio *)crp->crp_buf, crd->crd_skip, 451 crd->crd_len, abuf); 452 } else { 453 bcopy(abuf, crp->crp_buf + crd->crd_skip, crd->crd_len); 454 } 455 456 /* copy out last block for use as next session IV */ 457 if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) { 458 if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) { 459 m_copydata((struct mbuf *)crp->crp_buf, 460 crd->crd_skip + crd->crd_len - 16, 16, ses->ses_iv); 461 } else if ((crp->crp_flags & CRYPTO_F_IOV) != 0) { 462 cuio_copydata((struct uio *)crp->crp_buf, 463 crd->crd_skip + crd->crd_len - 16, 16, ses->ses_iv); 464 } else { 465 bcopy(crp->crp_buf + crd->crd_skip + crd->crd_len - 16, 466 ses->ses_iv, 16); 467 } 468 } 469 470out: 471 if (buf != NULL) { 472 bzero(buf, crd->crd_len + 16); 473 free(buf, M_DEVBUF); 474 } 475 crp->crp_etype = err; 476 crypto_done(crp); 477 return (err); 478} 479 480static int 481padlock_modevent(module_t mod, int type, void *unused __unused) 482{ 483 int error; 484 485 error = EOPNOTSUPP; 486 switch (type) { 487 case MOD_LOAD: 488 error = padlock_init(); 489 break; 490 case MOD_UNLOAD: 491 error = padlock_destroy(); 492 break; 493 } 494 return (error); 495} 496 497static moduledata_t padlock_mod = { 498 "padlock", 499 padlock_modevent, 500 0 501}; 502DECLARE_MODULE(padlock, padlock_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 503MODULE_VERSION(padlock, 1); 504MODULE_DEPEND(padlock, crypto, 1, 1, 1); 505