1/*-
| 1/*-
|
2 * Copyright (c) 2005 Pawel Jakub Dawidek 3 * Copyright (c) 2004 Mark R V Murray
| 2 * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
|
4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27
| 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26
|
28/* $OpenBSD: via.c,v 1.3 2004/06/15 23:36:55 deraadt Exp $ */ 29/*- 30 * Copyright (c) 2003 Jason Wright 31 * Copyright (c) 2003, 2004 Theo de Raadt 32 * All rights reserved. 33 * 34 * Permission to use, copy, modify, and distribute this software for any 35 * purpose with or without fee is hereby granted, provided that the above 36 * copyright notice and this permission notice appear in all copies. 37 * 38 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 39 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 40 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 41 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 42 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 43 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 44 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 45 */ 46
| |
47#include <sys/cdefs.h>
| 27#include <sys/cdefs.h>
|
48__FBSDID("$FreeBSD: head/sys/crypto/via/padlock.c 160573 2006-07-22 13:14:11Z pjd $");
| 28__FBSDID("$FreeBSD: head/sys/crypto/via/padlock.c 160582 2006-07-22 16:18:47Z pjd $");
|
49 50#include <sys/param.h> 51#include <sys/systm.h> 52#include <sys/kernel.h> 53#include <sys/module.h> 54#include <sys/lock.h> 55#include <sys/mutex.h> 56#include <sys/malloc.h> 57#include <sys/libkern.h> 58#if defined(__i386__) && !defined(PC98) 59#include <machine/cpufunc.h> 60#include <machine/cputypes.h> 61#include <machine/md_var.h> 62#include <machine/specialreg.h> 63#endif 64 65#include <opencrypto/cryptodev.h>
| 29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/kernel.h> 33#include <sys/module.h> 34#include <sys/lock.h> 35#include <sys/mutex.h> 36#include <sys/malloc.h> 37#include <sys/libkern.h> 38#if defined(__i386__) && !defined(PC98) 39#include <machine/cpufunc.h> 40#include <machine/cputypes.h> 41#include <machine/md_var.h> 42#include <machine/specialreg.h> 43#endif 44 45#include <opencrypto/cryptodev.h>
|
66#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */ 67#include <opencrypto/xform.h> 68#include <crypto/rijndael/rijndael.h>
| |
69
| 46
|
| 47#include <crypto/via/padlock.h>
|
70
| 48
|
71#define PADLOCK_ROUND_COUNT_AES128 10 72#define PADLOCK_ROUND_COUNT_AES192 12 73#define PADLOCK_ROUND_COUNT_AES256 14
| 49/* 50 * Technical documentation about the PadLock engine can be found here: 51 * 52 * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf 53 */
|
74
| 54
|
75#define PADLOCK_ALGORITHM_TYPE_AES 0 76 77#define PADLOCK_KEY_GENERATION_HW 0 78#define PADLOCK_KEY_GENERATION_SW 1 79 80#define PADLOCK_DIRECTION_ENCRYPT 0 81#define PADLOCK_DIRECTION_DECRYPT 1 82 83#define PADLOCK_KEY_SIZE_128 0 84#define PADLOCK_KEY_SIZE_192 1 85#define PADLOCK_KEY_SIZE_256 2 86 87union padlock_cw { 88 uint64_t raw; 89 struct { 90 u_int round_count : 4; 91 u_int algorithm_type : 3; 92 u_int key_generation : 1; 93 u_int intermediate : 1; 94 u_int direction : 1; 95 u_int key_size : 2; 96 u_int filler0 : 20; 97 u_int filler1 : 32; 98 u_int filler2 : 32; 99 u_int filler3 : 32; 100 } __field; 101}; 102#define cw_round_count __field.round_count 103#define cw_algorithm_type __field.algorithm_type 104#define cw_key_generation __field.key_generation 105#define cw_intermediate __field.intermediate 106#define cw_direction __field.direction 107#define cw_key_size __field.key_size 108#define cw_filler0 __field.filler0 109#define cw_filler1 __field.filler1 110#define cw_filler2 __field.filler2 111#define cw_filler3 __field.filler3 112 113struct padlock_session { 114 union padlock_cw ses_cw __aligned(16); 115 uint32_t ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */ 116 uint32_t ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */ 117 uint8_t ses_iv[16] __aligned(16); /* 128 bit aligned */ 118 struct auth_hash *ses_axf; 119 uint8_t *ses_ictx; 120 uint8_t *ses_octx; 121 int ses_mlen; 122 int ses_used; 123 uint32_t ses_id; 124 TAILQ_ENTRY(padlock_session) ses_next; 125}; 126
| |
127struct padlock_softc { 128 int32_t sc_cid; 129 uint32_t sc_sid; 130 TAILQ_HEAD(, padlock_session) sc_sessions; 131 struct mtx sc_sessions_mtx; 132}; 133 134static struct padlock_softc *padlock_sc; 135 136static int padlock_newsession(void *arg __unused, uint32_t *sidp, 137 struct cryptoini *cri); 138static int padlock_freesession(void *arg __unused, uint64_t tid); 139static int padlock_process(void *arg __unused, struct cryptop *crp, 140 int hint __unused); 141
| 55struct padlock_softc { 56 int32_t sc_cid; 57 uint32_t sc_sid; 58 TAILQ_HEAD(, padlock_session) sc_sessions; 59 struct mtx sc_sessions_mtx; 60}; 61 62static struct padlock_softc *padlock_sc; 63 64static int padlock_newsession(void *arg __unused, uint32_t *sidp, 65 struct cryptoini *cri); 66static int padlock_freesession(void *arg __unused, uint64_t tid); 67static int padlock_process(void *arg __unused, struct cryptop *crp, 68 int hint __unused); 69
|
142static __inline void 143padlock_cbc(void *in, void *out, size_t count, void *key, union padlock_cw *cw, 144 void *iv) 145{ 146#ifdef __GNUCLIKE_ASM 147 /* The .byte line is really VIA C3 "xcrypt-cbc" instruction */ 148 __asm __volatile( 149 "pushf \n\t" 150 "popf \n\t" 151 "rep \n\t" 152 ".byte 0x0f, 0xa7, 0xd0" 153 : "+a" (iv), "+c" (count), "+D" (out), "+S" (in) 154 : "b" (key), "d" (cw) 155 : "cc", "memory" 156 ); 157#endif 158}
| 70MALLOC_DEFINE(M_PADLOCK, "padlock_data", "PadLock Data");
|
159 160static int 161padlock_init(void) 162{ 163 struct padlock_softc *sc;
| 71 72static int 73padlock_init(void) 74{ 75 struct padlock_softc *sc;
|
| 76 char capp[256]; 77
|
164#if defined(__i386__) && !defined(PC98)
| 78#if defined(__i386__) && !defined(PC98)
|
| 79 /* If there is no AES support, we has nothing to do here. */
|
165 if (!(via_feature_xcrypt & VIA_HAS_AES)) {
| 80 if (!(via_feature_xcrypt & VIA_HAS_AES)) {
|
166 printf("PADLOCK: No ACE support.\n");
| 81 printf("PadLock: No ACE support.\n");
|
167 return (EINVAL);
| 82 return (EINVAL);
|
168 } else 169 printf("PADLOCK: HW support loaded.\n");
| 83 } 84 strlcpy(capp, "AES-CBC", sizeof(capp)); 85#if 0 86 strlcat(capp, ",AES-EBC", sizeof(capp)); 87 strlcat(capp, ",AES-CFB", sizeof(capp)); 88 strlcat(capp, ",AES-OFB", sizeof(capp)); 89#endif 90 if (via_feature_xcrypt & VIA_HAS_SHA) { 91 strlcat(capp, ",SHA1", sizeof(capp)); 92 strlcat(capp, ",SHA256", sizeof(capp)); 93 } 94#if 0 95 if (via_feature_xcrypt & VIA_HAS_AESCTR) 96 strlcat(capp, ",AES-CTR", sizeof(capp)); 97 if (via_feature_xcrypt & VIA_HAS_MM) 98 strlcat(capp, ",RSA", sizeof(capp)); 99#endif 100 printf("PadLock: HW support loaded for %s.\n", capp);
|
170#else 171 return (EINVAL); 172#endif 173
| 101#else 102 return (EINVAL); 103#endif 104
|
174 padlock_sc = sc = malloc(sizeof(*padlock_sc), M_DEVBUF,
| 105 padlock_sc = sc = malloc(sizeof(*padlock_sc), M_PADLOCK,
|
175 M_WAITOK | M_ZERO); 176 TAILQ_INIT(&sc->sc_sessions); 177 sc->sc_sid = 1; 178 179 sc->sc_cid = crypto_get_driverid(0); 180 if (sc->sc_cid < 0) {
| 106 M_WAITOK | M_ZERO); 107 TAILQ_INIT(&sc->sc_sessions); 108 sc->sc_sid = 1; 109 110 sc->sc_cid = crypto_get_driverid(0); 111 if (sc->sc_cid < 0) {
|
181 printf("PADLOCK: Could not get crypto driver id.\n"); 182 free(padlock_sc, M_DEVBUF);
| 112 printf("PadLock: Could not get crypto driver id.\n"); 113 free(padlock_sc, M_PADLOCK);
|
183 padlock_sc = NULL; 184 return (ENOMEM); 185 } 186 187 mtx_init(&sc->sc_sessions_mtx, "padlock_mtx", NULL, MTX_DEF); 188 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, padlock_newsession, 189 padlock_freesession, padlock_process, NULL); 190 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, padlock_newsession, 191 padlock_freesession, padlock_process, NULL); 192 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, padlock_newsession, 193 padlock_freesession, padlock_process, NULL); 194 crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0, 195 padlock_newsession, padlock_freesession, padlock_process, NULL); 196 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0, 197 padlock_newsession, padlock_freesession, padlock_process, NULL); 198 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0, 199 padlock_newsession, padlock_freesession, padlock_process, NULL); 200 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0, 201 padlock_newsession, padlock_freesession, padlock_process, NULL); 202 return (0); 203} 204 205static int 206padlock_destroy(void) 207{ 208 struct padlock_softc *sc = padlock_sc; 209 struct padlock_session *ses; 210 u_int active = 0; 211 212 if (sc == NULL) 213 return (0); 214 mtx_lock(&sc->sc_sessions_mtx); 215 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 216 if (ses->ses_used) 217 active++; 218 } 219 if (active > 0) { 220 mtx_unlock(&sc->sc_sessions_mtx);
| 114 padlock_sc = NULL; 115 return (ENOMEM); 116 } 117 118 mtx_init(&sc->sc_sessions_mtx, "padlock_mtx", NULL, MTX_DEF); 119 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, padlock_newsession, 120 padlock_freesession, padlock_process, NULL); 121 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, padlock_newsession, 122 padlock_freesession, padlock_process, NULL); 123 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, padlock_newsession, 124 padlock_freesession, padlock_process, NULL); 125 crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0, 126 padlock_newsession, padlock_freesession, padlock_process, NULL); 127 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0, 128 padlock_newsession, padlock_freesession, padlock_process, NULL); 129 crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0, 130 padlock_newsession, padlock_freesession, padlock_process, NULL); 131 crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0, 132 padlock_newsession, padlock_freesession, padlock_process, NULL); 133 return (0); 134} 135 136static int 137padlock_destroy(void) 138{ 139 struct padlock_softc *sc = padlock_sc; 140 struct padlock_session *ses; 141 u_int active = 0; 142 143 if (sc == NULL) 144 return (0); 145 mtx_lock(&sc->sc_sessions_mtx); 146 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 147 if (ses->ses_used) 148 active++; 149 } 150 if (active > 0) { 151 mtx_unlock(&sc->sc_sessions_mtx);
|
221 printf("PADLOCK: Cannot destroy, %u sessions active.\n",
| 152 printf("PadLock: Cannot destroy, %u sessions active.\n",
|
222 active); 223 return (EBUSY); 224 } 225 padlock_sc = NULL; 226 for (ses = TAILQ_FIRST(&sc->sc_sessions); ses != NULL; 227 ses = TAILQ_FIRST(&sc->sc_sessions)) { 228 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
| 153 active); 154 return (EBUSY); 155 } 156 padlock_sc = NULL; 157 for (ses = TAILQ_FIRST(&sc->sc_sessions); ses != NULL; 158 ses = TAILQ_FIRST(&sc->sc_sessions)) { 159 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
|
229 free(ses, M_DEVBUF);
| 160 free(ses, M_PADLOCK);
|
230 } 231 mtx_destroy(&sc->sc_sessions_mtx); 232 crypto_unregister_all(sc->sc_cid);
| 161 } 162 mtx_destroy(&sc->sc_sessions_mtx); 163 crypto_unregister_all(sc->sc_cid);
|
233 free(sc, M_DEVBUF);
| 164 free(sc, M_PADLOCK);
|
234 return (0); 235} 236
| 165 return (0); 166} 167
|
237static void 238padlock_setup_enckey(struct padlock_session *ses, caddr_t key, int klen) 239{ 240 union padlock_cw *cw; 241 int i; 242 243 cw = &ses->ses_cw; 244 if (cw->cw_key_generation == PADLOCK_KEY_GENERATION_SW) { 245 /* Build expanded keys for both directions */ 246 rijndaelKeySetupEnc(ses->ses_ekey, key, klen); 247 rijndaelKeySetupDec(ses->ses_dkey, key, klen); 248 for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) { 249 ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]); 250 ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]); 251 } 252 } else { 253 bcopy(key, ses->ses_ekey, klen); 254 bcopy(key, ses->ses_dkey, klen); 255 } 256} 257 258static void 259padlock_setup_mackey(struct padlock_session *ses, caddr_t key, int klen) 260{ 261 struct auth_hash *axf; 262 int i; 263 264 klen /= 8; 265 axf = ses->ses_axf; 266 267 for (i = 0; i < klen; i++) 268 key[i] ^= HMAC_IPAD_VAL; 269 270 axf->Init(ses->ses_ictx); 271 axf->Update(ses->ses_ictx, key, klen); 272 axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen); 273 274 for (i = 0; i < klen; i++) 275 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 276 277 axf->Init(ses->ses_octx); 278 axf->Update(ses->ses_octx, key, klen); 279 axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen); 280 281 for (i = 0; i < klen; i++) 282 key[i] ^= HMAC_OPAD_VAL; 283} 284 285/* 286 * Compute keyed-hash authenticator. 287 */
| |
288static int
| 168static int
|
289padlock_authcompute(struct padlock_session *ses, struct cryptodesc *crd, 290 caddr_t buf, int flags) 291{ 292 u_char hash[HASH_MAX_LEN]; 293 struct auth_hash *axf; 294 union authctx ctx; 295 int error; 296 297 axf = ses->ses_axf; 298 299 bcopy(ses->ses_ictx, &ctx, axf->ctxsize); 300 301 error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len, 302 (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx); 303 if (error != 0) 304 return (error); 305 306 axf->Final(hash, &ctx); 307 bcopy(ses->ses_octx, &ctx, axf->ctxsize); 308 axf->Update(&ctx, hash, axf->hashsize); 309 axf->Final(hash, &ctx); 310 311 /* Inject the authentication data */ 312 crypto_copyback(flags, buf, crd->crd_inject, 313 ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash); 314 return (0); 315} 316 317 318static int
| |
319padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri) 320{ 321 struct padlock_softc *sc = padlock_sc; 322 struct padlock_session *ses = NULL; 323 struct cryptoini *encini, *macini;
| 169padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri) 170{ 171 struct padlock_softc *sc = padlock_sc; 172 struct padlock_session *ses = NULL; 173 struct cryptoini *encini, *macini;
|
324 union padlock_cw *cw;
| 174 int error;
|
325 326 if (sc == NULL || sidp == NULL || cri == NULL) 327 return (EINVAL); 328 329 encini = macini = NULL; 330 for (; cri != NULL; cri = cri->cri_next) { 331 switch (cri->cri_alg) { 332 case CRYPTO_NULL_HMAC: 333 case CRYPTO_MD5_HMAC: 334 case CRYPTO_SHA1_HMAC: 335 case CRYPTO_RIPEMD160_HMAC: 336 case CRYPTO_SHA2_256_HMAC: 337 case CRYPTO_SHA2_384_HMAC: 338 case CRYPTO_SHA2_512_HMAC: 339 if (macini != NULL) 340 return (EINVAL); 341 macini = cri; 342 break; 343 case CRYPTO_AES_CBC: 344 if (encini != NULL) 345 return (EINVAL); 346 encini = cri; 347 break; 348 default: 349 return (EINVAL); 350 } 351 } 352 353 /* 354 * We only support HMAC algorithms to be able to work with 355 * fast_ipsec(4), so if we are asked only for authentication without 356 * encryption, don't pretend we can accellerate it. 357 */ 358 if (encini == NULL) 359 return (EINVAL);
| 175 176 if (sc == NULL || sidp == NULL || cri == NULL) 177 return (EINVAL); 178 179 encini = macini = NULL; 180 for (; cri != NULL; cri = cri->cri_next) { 181 switch (cri->cri_alg) { 182 case CRYPTO_NULL_HMAC: 183 case CRYPTO_MD5_HMAC: 184 case CRYPTO_SHA1_HMAC: 185 case CRYPTO_RIPEMD160_HMAC: 186 case CRYPTO_SHA2_256_HMAC: 187 case CRYPTO_SHA2_384_HMAC: 188 case CRYPTO_SHA2_512_HMAC: 189 if (macini != NULL) 190 return (EINVAL); 191 macini = cri; 192 break; 193 case CRYPTO_AES_CBC: 194 if (encini != NULL) 195 return (EINVAL); 196 encini = cri; 197 break; 198 default: 199 return (EINVAL); 200 } 201 } 202 203 /* 204 * We only support HMAC algorithms to be able to work with 205 * fast_ipsec(4), so if we are asked only for authentication without 206 * encryption, don't pretend we can accellerate it. 207 */ 208 if (encini == NULL) 209 return (EINVAL);
|
360 if (encini->cri_klen != 128 && encini->cri_klen != 192 && 361 encini->cri_klen != 256) { 362 return (EINVAL); 363 }
| |
364 365 /* 366 * Let's look for a free session structure. 367 */ 368 mtx_lock(&sc->sc_sessions_mtx); 369 /* 370 * Free sessions goes first, so if first session is used, we need to 371 * allocate one. 372 */ 373 ses = TAILQ_FIRST(&sc->sc_sessions); 374 if (ses == NULL || ses->ses_used) 375 ses = NULL; 376 else { 377 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 378 ses->ses_used = 1; 379 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 380 } 381 mtx_unlock(&sc->sc_sessions_mtx); 382 if (ses == NULL) {
| 210 211 /* 212 * Let's look for a free session structure. 213 */ 214 mtx_lock(&sc->sc_sessions_mtx); 215 /* 216 * Free sessions goes first, so if first session is used, we need to 217 * allocate one. 218 */ 219 ses = TAILQ_FIRST(&sc->sc_sessions); 220 if (ses == NULL || ses->ses_used) 221 ses = NULL; 222 else { 223 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next); 224 ses->ses_used = 1; 225 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 226 } 227 mtx_unlock(&sc->sc_sessions_mtx); 228 if (ses == NULL) {
|
383 ses = malloc(sizeof(*ses), M_DEVBUF, M_NOWAIT | M_ZERO);
| 229 ses = malloc(sizeof(*ses), M_PADLOCK, M_NOWAIT | M_ZERO);
|
384 if (ses == NULL) 385 return (ENOMEM); 386 ses->ses_used = 1; 387 mtx_lock(&sc->sc_sessions_mtx); 388 ses->ses_id = sc->sc_sid++; 389 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 390 mtx_unlock(&sc->sc_sessions_mtx); 391 } 392
| 230 if (ses == NULL) 231 return (ENOMEM); 232 ses->ses_used = 1; 233 mtx_lock(&sc->sc_sessions_mtx); 234 ses->ses_id = sc->sc_sid++; 235 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 236 mtx_unlock(&sc->sc_sessions_mtx); 237 } 238
|
393 cw = &ses->ses_cw; 394 bzero(cw, sizeof(*cw)); 395 cw->cw_algorithm_type = PADLOCK_ALGORITHM_TYPE_AES; 396 cw->cw_key_generation = PADLOCK_KEY_GENERATION_SW; 397 cw->cw_intermediate = 0; 398 switch (encini->cri_klen) { 399 case 128: 400 cw->cw_round_count = PADLOCK_ROUND_COUNT_AES128; 401 cw->cw_key_size = PADLOCK_KEY_SIZE_128; 402#ifdef HW_KEY_GENERATION 403 /* This doesn't buy us much, that's why it is commented out. */ 404 cw->cw_key_generation = PADLOCK_KEY_GENERATION_HW; 405#endif 406 break; 407 case 192: 408 cw->cw_round_count = PADLOCK_ROUND_COUNT_AES192; 409 cw->cw_key_size = PADLOCK_KEY_SIZE_192; 410 break; 411 case 256: 412 cw->cw_round_count = PADLOCK_ROUND_COUNT_AES256; 413 cw->cw_key_size = PADLOCK_KEY_SIZE_256; 414 break;
| 239 error = padlock_cipher_setup(ses, encini); 240 if (error != 0) { 241 padlock_freesession(NULL, ses->ses_id); 242 return (error);
|
415 }
| 243 }
|
416 if (encini->cri_key != NULL) 417 padlock_setup_enckey(ses, encini->cri_key, encini->cri_klen);
| |
418
| 244
|
419 arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0); 420
| |
421 if (macini != NULL) {
| 245 if (macini != NULL) {
|
422 ses->ses_mlen = macini->cri_mlen; 423 424 /* Find software structure which describes HMAC algorithm. */ 425 switch (macini->cri_alg) { 426 case CRYPTO_NULL_HMAC: 427 ses->ses_axf = &auth_hash_null; 428 break; 429 case CRYPTO_MD5_HMAC: 430 ses->ses_axf = &auth_hash_hmac_md5; 431 break; 432 case CRYPTO_SHA1_HMAC: 433 ses->ses_axf = &auth_hash_hmac_sha1; 434 break; 435 case CRYPTO_RIPEMD160_HMAC: 436 ses->ses_axf = &auth_hash_hmac_ripemd_160; 437 break; 438 case CRYPTO_SHA2_256_HMAC: 439 ses->ses_axf = &auth_hash_hmac_sha2_256; 440 break; 441 case CRYPTO_SHA2_384_HMAC: 442 ses->ses_axf = &auth_hash_hmac_sha2_384; 443 break; 444 case CRYPTO_SHA2_512_HMAC: 445 ses->ses_axf = &auth_hash_hmac_sha2_512; 446 break; 447 } 448 449 /* Allocate memory for HMAC inner and outer contexts. */ 450 ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_CRYPTO_DATA, 451 M_NOWAIT); 452 ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_CRYPTO_DATA, 453 M_NOWAIT); 454 if (ses->ses_ictx == NULL || ses->ses_octx == NULL) {
| 246 error = padlock_hash_setup(ses, macini); 247 if (error != 0) {
|
455 padlock_freesession(NULL, ses->ses_id);
| 248 padlock_freesession(NULL, ses->ses_id);
|
456 return (ENOMEM);
| 249 return (error);
|
457 }
| 250 }
|
458 459 /* Setup key if given. */ 460 if (macini->cri_key != NULL) { 461 padlock_setup_mackey(ses, macini->cri_key, 462 macini->cri_klen); 463 }
| |
464 } 465 466 *sidp = ses->ses_id; 467 return (0); 468} 469 470static int 471padlock_freesession(void *arg __unused, uint64_t tid) 472{ 473 struct padlock_softc *sc = padlock_sc; 474 struct padlock_session *ses; 475 uint32_t sid = ((uint32_t)tid) & 0xffffffff; 476 477 if (sc == NULL) 478 return (EINVAL); 479 mtx_lock(&sc->sc_sessions_mtx); 480 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 481 if (ses->ses_id == sid) 482 break; 483 } 484 if (ses == NULL) { 485 mtx_unlock(&sc->sc_sessions_mtx); 486 return (EINVAL); 487 } 488 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
| 251 } 252 253 *sidp = ses->ses_id; 254 return (0); 255} 256 257static int 258padlock_freesession(void *arg __unused, uint64_t tid) 259{ 260 struct padlock_softc *sc = padlock_sc; 261 struct padlock_session *ses; 262 uint32_t sid = ((uint32_t)tid) & 0xffffffff; 263 264 if (sc == NULL) 265 return (EINVAL); 266 mtx_lock(&sc->sc_sessions_mtx); 267 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 268 if (ses->ses_id == sid) 269 break; 270 } 271 if (ses == NULL) { 272 mtx_unlock(&sc->sc_sessions_mtx); 273 return (EINVAL); 274 } 275 TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
|
489 if (ses->ses_ictx != NULL) { 490 bzero(ses->ses_ictx, ses->ses_axf->ctxsize); 491 free(ses->ses_ictx, M_CRYPTO_DATA); 492 ses->ses_ictx = NULL; 493 } 494 if (ses->ses_octx != NULL) { 495 bzero(ses->ses_ictx, ses->ses_axf->ctxsize); 496 free(ses->ses_octx, M_CRYPTO_DATA); 497 ses->ses_octx = NULL; 498 }
| 276 padlock_hash_free(ses);
|
499 bzero(ses, sizeof(*ses)); 500 ses->ses_used = 0; 501 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 502 mtx_unlock(&sc->sc_sessions_mtx); 503 return (0); 504} 505 506static int 507padlock_process(void *arg __unused, struct cryptop *crp, int hint __unused) 508{ 509 struct padlock_softc *sc = padlock_sc;
| 277 bzero(ses, sizeof(*ses)); 278 ses->ses_used = 0; 279 TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next); 280 mtx_unlock(&sc->sc_sessions_mtx); 281 return (0); 282} 283 284static int 285padlock_process(void *arg __unused, struct cryptop *crp, int hint __unused) 286{ 287 struct padlock_softc *sc = padlock_sc;
|
510 struct padlock_session *ses; 511 union padlock_cw *cw;
| 288 struct padlock_session *ses = NULL;
|
512 struct cryptodesc *crd, *enccrd, *maccrd;
| 289 struct cryptodesc *crd, *enccrd, *maccrd;
|
513 uint32_t *key; 514 u_char *buf, *abuf;
| |
515 int error = 0; 516 517 enccrd = maccrd = NULL;
| 290 int error = 0; 291 292 enccrd = maccrd = NULL;
|
518 buf = NULL;
| |
519 520 if (crp == NULL || crp->crp_callback == NULL || crp->crp_desc == NULL) { 521 error = EINVAL; 522 goto out; 523 } 524 525 for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { 526 switch (crd->crd_alg) { 527 case CRYPTO_NULL_HMAC: 528 case CRYPTO_MD5_HMAC: 529 case CRYPTO_SHA1_HMAC: 530 case CRYPTO_RIPEMD160_HMAC: 531 case CRYPTO_SHA2_256_HMAC: 532 case CRYPTO_SHA2_384_HMAC: 533 case CRYPTO_SHA2_512_HMAC: 534 if (maccrd != NULL) { 535 error = EINVAL; 536 goto out; 537 } 538 maccrd = crd; 539 break; 540 case CRYPTO_AES_CBC: 541 if (enccrd != NULL) { 542 error = EINVAL; 543 goto out; 544 } 545 enccrd = crd; 546 break; 547 default: 548 return (EINVAL); 549 } 550 } 551 if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { 552 error = EINVAL; 553 goto out; 554 } 555 556 mtx_lock(&sc->sc_sessions_mtx); 557 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 558 if (ses->ses_id == (crp->crp_sid & 0xffffffff)) 559 break; 560 } 561 mtx_unlock(&sc->sc_sessions_mtx); 562 if (ses == NULL) { 563 error = EINVAL; 564 goto out; 565 } 566
| 293 294 if (crp == NULL || crp->crp_callback == NULL || crp->crp_desc == NULL) { 295 error = EINVAL; 296 goto out; 297 } 298 299 for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { 300 switch (crd->crd_alg) { 301 case CRYPTO_NULL_HMAC: 302 case CRYPTO_MD5_HMAC: 303 case CRYPTO_SHA1_HMAC: 304 case CRYPTO_RIPEMD160_HMAC: 305 case CRYPTO_SHA2_256_HMAC: 306 case CRYPTO_SHA2_384_HMAC: 307 case CRYPTO_SHA2_512_HMAC: 308 if (maccrd != NULL) { 309 error = EINVAL; 310 goto out; 311 } 312 maccrd = crd; 313 break; 314 case CRYPTO_AES_CBC: 315 if (enccrd != NULL) { 316 error = EINVAL; 317 goto out; 318 } 319 enccrd = crd; 320 break; 321 default: 322 return (EINVAL); 323 } 324 } 325 if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { 326 error = EINVAL; 327 goto out; 328 } 329 330 mtx_lock(&sc->sc_sessions_mtx); 331 TAILQ_FOREACH(ses, &sc->sc_sessions, ses_next) { 332 if (ses->ses_id == (crp->crp_sid & 0xffffffff)) 333 break; 334 } 335 mtx_unlock(&sc->sc_sessions_mtx); 336 if (ses == NULL) { 337 error = EINVAL; 338 goto out; 339 } 340
|
567 buf = malloc(enccrd->crd_len + 16, M_DEVBUF, M_NOWAIT); 568 if (buf == NULL) { 569 error = ENOMEM; 570 goto out; 571 } 572 /* Buffer has to be 16 bytes aligned. */ 573 abuf = buf + 16 - ((uintptr_t)buf % 16); 574 575 if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) 576 padlock_setup_enckey(ses, enccrd->crd_key, enccrd->crd_klen); 577 if (maccrd != NULL && (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) 578 padlock_setup_mackey(ses, maccrd->crd_key, maccrd->crd_klen); 579 580 cw = &ses->ses_cw; 581 cw->cw_filler0 = 0; 582 cw->cw_filler1 = 0; 583 cw->cw_filler2 = 0; 584 cw->cw_filler3 = 0; 585 if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { 586 cw->cw_direction = PADLOCK_DIRECTION_ENCRYPT; 587 key = ses->ses_ekey; 588 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 589 bcopy(enccrd->crd_iv, ses->ses_iv, 16); 590 591 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 592 crypto_copyback(crp->crp_flags, crp->crp_buf, 593 enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv); 594 } 595 } else { 596 cw->cw_direction = PADLOCK_DIRECTION_DECRYPT; 597 key = ses->ses_dkey; 598 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) 599 bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN); 600 else { 601 crypto_copydata(crp->crp_flags, crp->crp_buf, 602 enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv); 603 } 604 } 605
| |
606 /* Perform data authentication if requested before encryption. */ 607 if (maccrd != NULL && maccrd->crd_next == enccrd) {
| 341 /* Perform data authentication if requested before encryption. */ 342 if (maccrd != NULL && maccrd->crd_next == enccrd) {
|
608 error = padlock_authcompute(ses, maccrd, crp->crp_buf, 609 crp->crp_flags);
| 343 error = padlock_hash_process(ses, maccrd, crp);
|
610 if (error != 0) 611 goto out; 612 } 613
| 344 if (error != 0) 345 goto out; 346 } 347
|
614 crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 615 enccrd->crd_len, abuf);
| 348 error = padlock_cipher_process(ses, enccrd, crp); 349 if (error != 0) 350 goto out;
|
616
| 351
|
617 padlock_cbc(abuf, abuf, enccrd->crd_len / 16, key, cw, ses->ses_iv); 618 619 crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, 620 enccrd->crd_len, abuf); 621
| |
622 /* Perform data authentication if requested after encryption. */ 623 if (maccrd != NULL && enccrd->crd_next == maccrd) {
| 352 /* Perform data authentication if requested after encryption. */ 353 if (maccrd != NULL && enccrd->crd_next == maccrd) {
|
624 error = padlock_authcompute(ses, maccrd, crp->crp_buf, 625 crp->crp_flags);
| 354 error = padlock_hash_process(ses, maccrd, crp);
|
626 if (error != 0) 627 goto out; 628 } 629
| 355 if (error != 0) 356 goto out; 357 } 358
|
630 /* copy out last block for use as next session IV */ 631 if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { 632 crypto_copydata(crp->crp_flags, crp->crp_buf, 633 enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, 634 AES_BLOCK_LEN, ses->ses_iv); 635 } 636
| |
637out:
| 359out:
|
638 if (buf != NULL) { 639 bzero(buf, enccrd->crd_len + 16); 640 free(buf, M_DEVBUF);
| 360#if 0 361 /* 362 * This code is not necessary, because contexts will be freed on next 363 * padlock_setup_mackey() call or at padlock_freesession() call. 364 */ 365 if (ses != NULL && maccrd != NULL && 366 (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { 367 padlock_free_ctx(ses->ses_axf, ses->ses_ictx); 368 padlock_free_ctx(ses->ses_axf, ses->ses_octx);
|
641 }
| 369 }
|
| 370#endif
|
642 crp->crp_etype = error; 643 crypto_done(crp); 644 return (error); 645} 646 647static int 648padlock_modevent(module_t mod, int type, void *unused __unused) 649{ 650 int error; 651 652 error = EOPNOTSUPP; 653 switch (type) { 654 case MOD_LOAD: 655 error = padlock_init(); 656 break; 657 case MOD_UNLOAD: 658 error = padlock_destroy(); 659 break; 660 } 661 return (error); 662} 663 664static moduledata_t padlock_mod = { 665 "padlock", 666 padlock_modevent, 667 0 668}; 669DECLARE_MODULE(padlock, padlock_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 670MODULE_VERSION(padlock, 1); 671MODULE_DEPEND(padlock, crypto, 1, 1, 1);
| 371 crp->crp_etype = error; 372 crypto_done(crp); 373 return (error); 374} 375 376static int 377padlock_modevent(module_t mod, int type, void *unused __unused) 378{ 379 int error; 380 381 error = EOPNOTSUPP; 382 switch (type) { 383 case MOD_LOAD: 384 error = padlock_init(); 385 break; 386 case MOD_UNLOAD: 387 error = padlock_destroy(); 388 break; 389 } 390 return (error); 391} 392 393static moduledata_t padlock_mod = { 394 "padlock", 395 padlock_modevent, 396 0 397}; 398DECLARE_MODULE(padlock, padlock_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 399MODULE_VERSION(padlock, 1); 400MODULE_DEPEND(padlock, crypto, 1, 1, 1);
|