1/* $OpenBSD: via.c,v 1.50 2021/10/24 10:26:22 patrick Exp $ */ 2/* $NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $ */ 3 4/*- 5 * Copyright (c) 2003 Jason Wright 6 * Copyright (c) 2003, 2004 Theo de Raadt 7 * All rights reserved. 8 * 9 * Permission to use, copy, modify, and distribute this software for any 10 * purpose with or without fee is hereby granted, provided that the above 11 * copyright notice and this permission notice appear in all copies. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 20 */ 21 22#include <sys/param.h> 23#include <sys/systm.h> 24#include <sys/timeout.h> 25#include <sys/malloc.h> 26#include <sys/mbuf.h> 27 28#ifdef CRYPTO 29#include <crypto/cryptodev.h> 30#include <crypto/aes.h> 31#include <crypto/xform.h> 32#include <crypto/cryptosoft.h> 33#endif 34 35#include <machine/cpufunc.h> 36#include <machine/specialreg.h> 37 38void viac3_rnd(void *); 39 40 41#ifdef CRYPTO 42 43struct viac3_session { 44 u_int32_t ses_ekey[4 * (AES_MAXROUNDS + 1) + 4]; /* 128 bit aligned */ 45 u_int32_t ses_dkey[4 * (AES_MAXROUNDS + 1) + 4]; /* 128 bit aligned */ 46 u_int32_t ses_cw0; 47 struct swcr_data *swd; 48 int ses_klen; 49 int ses_used; 50}; 51 52struct viac3_softc { 53 u_int32_t op_cw[4]; /* 128 bit aligned */ 54 u_int8_t op_iv[16]; /* 128 bit aligned */ 55 void *op_buf; 56 57 /* normal softc stuff */ 58 int32_t sc_cid; 59 int sc_nsessions; 60 struct viac3_session *sc_sessions; 61}; 62 63#define VIAC3_SESSION(sid) ((sid) & 0x0fffffff) 64#define VIAC3_SID(crd,ses) (((crd) << 28) | ((ses) & 0x0fffffff)) 65 66static struct viac3_softc *vc3_sc; 67extern int i386_has_xcrypt; 68 69extern const u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN]; 70extern const u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN]; 71 72void viac3_crypto_setup(void); 73int viac3_crypto_newsession(u_int32_t *, struct cryptoini *); 74int viac3_crypto_process(struct cryptop *); 75int viac3_crypto_swauth(struct cryptop *, struct cryptodesc *, 76 struct swcr_data *, caddr_t); 77int viac3_crypto_encdec(struct cryptop *, struct cryptodesc *, 78 struct viac3_session *, struct viac3_softc *, caddr_t); 79int viac3_crypto_freesession(u_int64_t); 80static __inline void viac3_cbc(void *, void *, void *, void *, int, void *); 81 82void 83viac3_crypto_setup(void) 84{ 85 int algs[CRYPTO_ALGORITHM_MAX + 1]; 86 87 vc3_sc = malloc(sizeof(*vc3_sc), M_DEVBUF, M_NOWAIT|M_ZERO); 88 if (vc3_sc == NULL) 89 return; 90 91 bzero(algs, sizeof(algs)); 92 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED; 93 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 94 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 95 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 96 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 97 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 98 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 99 algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED; 100 101 vc3_sc->sc_cid = crypto_get_driverid(0); 102 if (vc3_sc->sc_cid < 0) { 103 free(vc3_sc, M_DEVBUF, sizeof(*vc3_sc)); 104 vc3_sc = NULL; 105 return; 106 } 107 108 crypto_register(vc3_sc->sc_cid, algs, viac3_crypto_newsession, 109 viac3_crypto_freesession, viac3_crypto_process); 110} 111 112int 113viac3_crypto_newsession(u_int32_t *sidp, struct cryptoini *cri) 114{ 115 struct cryptoini *c; 116 struct viac3_softc *sc = vc3_sc; 117 struct viac3_session *ses = NULL; 118 const struct auth_hash *axf; 119 struct swcr_data *swd; 120 int sesn, i, cw0; 121 122 if (sc == NULL || sidp == NULL || cri == NULL) 123 return (EINVAL); 124 125 if (sc->sc_sessions == NULL) { 126 ses = sc->sc_sessions = malloc(sizeof(*ses), M_DEVBUF, 127 M_NOWAIT); 128 if (ses == NULL) 129 return (ENOMEM); 130 sesn = 0; 131 sc->sc_nsessions = 1; 132 } else { 133 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 134 if (sc->sc_sessions[sesn].ses_used == 0) { 135 ses = &sc->sc_sessions[sesn]; 136 break; 137 } 138 } 139 140 if (ses == NULL) { 141 sesn = sc->sc_nsessions; 142 ses = mallocarray(sesn + 1, sizeof(*ses), M_DEVBUF, 143 M_NOWAIT); 144 if (ses == NULL) 145 return (ENOMEM); 146 memcpy(ses, sc->sc_sessions, sesn * sizeof(*ses)); 147 explicit_bzero(sc->sc_sessions, sesn * sizeof(*ses)); 148 free(sc->sc_sessions, M_DEVBUF, sesn * sizeof(*ses)); 149 sc->sc_sessions = ses; 150 ses = &sc->sc_sessions[sesn]; 151 sc->sc_nsessions++; 152 } 153 } 154 155 bzero(ses, sizeof(*ses)); 156 ses->ses_used = 1; 157 158 for (c = cri; c != NULL; c = c->cri_next) { 159 switch (c->cri_alg) { 160 case CRYPTO_AES_CBC: 161 switch (c->cri_klen) { 162 case 128: 163 cw0 = C3_CRYPT_CWLO_KEY128; 164 break; 165 case 192: 166 cw0 = C3_CRYPT_CWLO_KEY192; 167 break; 168 case 256: 169 cw0 = C3_CRYPT_CWLO_KEY256; 170 break; 171 default: 172 viac3_crypto_freesession(sesn); 173 return (EINVAL); 174 } 175 cw0 |= C3_CRYPT_CWLO_ALG_AES | C3_CRYPT_CWLO_KEYGEN_SW | 176 C3_CRYPT_CWLO_NORMAL; 177 178 ses->ses_klen = c->cri_klen; 179 ses->ses_cw0 = cw0; 180 181 /* Build expanded keys for both directions */ 182 AES_KeySetup_Encrypt(ses->ses_ekey, c->cri_key, 183 c->cri_klen / 8); 184 AES_KeySetup_Decrypt(ses->ses_dkey, c->cri_key, 185 c->cri_klen / 8); 186 for (i = 0; i < 4 * (AES_MAXROUNDS + 1); i++) { 187 ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]); 188 ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]); 189 } 190 191 break; 192 193 case CRYPTO_MD5_HMAC: 194 axf = &auth_hash_hmac_md5_96; 195 goto authcommon; 196 case CRYPTO_SHA1_HMAC: 197 axf = &auth_hash_hmac_sha1_96; 198 goto authcommon; 199 case CRYPTO_RIPEMD160_HMAC: 200 axf = &auth_hash_hmac_ripemd_160_96; 201 goto authcommon; 202 case CRYPTO_SHA2_256_HMAC: 203 axf = &auth_hash_hmac_sha2_256_128; 204 goto authcommon; 205 case CRYPTO_SHA2_384_HMAC: 206 axf = &auth_hash_hmac_sha2_384_192; 207 goto authcommon; 208 case CRYPTO_SHA2_512_HMAC: 209 axf = &auth_hash_hmac_sha2_512_256; 210 authcommon: 211 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 212 M_NOWAIT|M_ZERO); 213 if (swd == NULL) { 214 viac3_crypto_freesession(sesn); 215 return (ENOMEM); 216 } 217 ses->swd = swd; 218 219 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 220 M_NOWAIT); 221 if (swd->sw_ictx == NULL) { 222 viac3_crypto_freesession(sesn); 223 return (ENOMEM); 224 } 225 226 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 227 M_NOWAIT); 228 if (swd->sw_octx == NULL) { 229 viac3_crypto_freesession(sesn); 230 return (ENOMEM); 231 } 232 233 for (i = 0; i < c->cri_klen / 8; i++) 234 c->cri_key[i] ^= HMAC_IPAD_VAL; 235 236 axf->Init(swd->sw_ictx); 237 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 238 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 239 axf->blocksize - (c->cri_klen / 8)); 240 241 for (i = 0; i < c->cri_klen / 8; i++) 242 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 243 HMAC_OPAD_VAL); 244 245 axf->Init(swd->sw_octx); 246 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 247 axf->Update(swd->sw_octx, hmac_opad_buffer, 248 axf->blocksize - (c->cri_klen / 8)); 249 250 for (i = 0; i < c->cri_klen / 8; i++) 251 c->cri_key[i] ^= HMAC_OPAD_VAL; 252 253 swd->sw_axf = axf; 254 swd->sw_alg = c->cri_alg; 255 256 break; 257 case CRYPTO_ESN: 258 /* nothing to do */ 259 break; 260 default: 261 viac3_crypto_freesession(sesn); 262 return (EINVAL); 263 } 264 } 265 266 *sidp = VIAC3_SID(0, sesn); 267 return (0); 268} 269 270int 271viac3_crypto_freesession(u_int64_t tid) 272{ 273 struct viac3_softc *sc = vc3_sc; 274 struct swcr_data *swd; 275 const struct auth_hash *axf; 276 int sesn; 277 u_int32_t sid = ((u_int32_t)tid) & 0xffffffff; 278 279 if (sc == NULL) 280 return (EINVAL); 281 sesn = VIAC3_SESSION(sid); 282 if (sesn >= sc->sc_nsessions) 283 return (EINVAL); 284 285 if (sc->sc_sessions[sesn].swd) { 286 swd = sc->sc_sessions[sesn].swd; 287 axf = swd->sw_axf; 288 289 if (swd->sw_ictx) { 290 explicit_bzero(swd->sw_ictx, axf->ctxsize); 291 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize); 292 } 293 if (swd->sw_octx) { 294 explicit_bzero(swd->sw_octx, axf->ctxsize); 295 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize); 296 } 297 free(swd, M_CRYPTO_DATA, sizeof(*swd)); 298 } 299 300 explicit_bzero(&sc->sc_sessions[sesn], sizeof(sc->sc_sessions[sesn])); 301 return (0); 302} 303 304static __inline void 305viac3_cbc(void *cw, void *src, void *dst, void *key, int rep, 306 void *iv) 307{ 308 unsigned int creg0; 309 310 creg0 = rcr0(); /* Permit access to SIMD/FPU path */ 311 lcr0(creg0 & ~(CR0_EM|CR0_TS)); 312 313 /* Do the deed */ 314 __asm volatile("pushfl; popfl"); 315 __asm volatile("rep xcryptcbc" : 316 : "a" (iv), "b" (key), "c" (rep), "d" (cw), "S" (src), "D" (dst) 317 : "memory", "cc"); 318 319 lcr0(creg0); 320} 321 322int 323viac3_crypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 324 struct swcr_data *sw, caddr_t buf) 325{ 326 int type; 327 328 if (crp->crp_flags & CRYPTO_F_IMBUF) 329 type = CRYPTO_BUF_MBUF; 330 else 331 type= CRYPTO_BUF_IOV; 332 333 return (swcr_authcompute(crp, crd, sw, buf, type)); 334} 335 336int 337viac3_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd, 338 struct viac3_session *ses, struct viac3_softc *sc, caddr_t buf) 339{ 340 u_int32_t *key; 341 int err = 0; 342 343 if ((crd->crd_len % 16) != 0) 344 return (EINVAL); 345 346 sc->op_buf = malloc(crd->crd_len, M_DEVBUF, M_NOWAIT); 347 if (sc->op_buf == NULL) 348 return (ENOMEM); 349 350 if (crd->crd_flags & CRD_F_ENCRYPT) { 351 sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_ENCRYPT; 352 key = ses->ses_ekey; 353 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 354 memcpy(sc->op_iv, crd->crd_iv, 16); 355 else 356 arc4random_buf(sc->op_iv, 16); 357 358 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 359 if (crp->crp_flags & CRYPTO_F_IMBUF) 360 err = m_copyback((struct mbuf *)crp->crp_buf, 361 crd->crd_inject, 16, sc->op_iv, M_NOWAIT); 362 else if (crp->crp_flags & CRYPTO_F_IOV) 363 cuio_copyback((struct uio *)crp->crp_buf, 364 crd->crd_inject, 16, sc->op_iv); 365 else 366 memcpy(crp->crp_buf + crd->crd_inject, 367 sc->op_iv, 16); 368 if (err) 369 goto errout; 370 } 371 } else { 372 sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_DECRYPT; 373 key = ses->ses_dkey; 374 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 375 memcpy(sc->op_iv, crd->crd_iv, 16); 376 else { 377 if (crp->crp_flags & CRYPTO_F_IMBUF) 378 m_copydata((struct mbuf *)crp->crp_buf, 379 crd->crd_inject, 16, sc->op_iv); 380 else if (crp->crp_flags & CRYPTO_F_IOV) 381 cuio_copydata((struct uio *)crp->crp_buf, 382 crd->crd_inject, 16, sc->op_iv); 383 else 384 memcpy(sc->op_iv, 385 crp->crp_buf + crd->crd_inject, 16); 386 } 387 } 388 389 if (crp->crp_flags & CRYPTO_F_IMBUF) 390 m_copydata((struct mbuf *)crp->crp_buf, 391 crd->crd_skip, crd->crd_len, sc->op_buf); 392 else if (crp->crp_flags & CRYPTO_F_IOV) 393 cuio_copydata((struct uio *)crp->crp_buf, 394 crd->crd_skip, crd->crd_len, sc->op_buf); 395 else 396 memcpy(sc->op_buf, crp->crp_buf + crd->crd_skip, crd->crd_len); 397 398 sc->op_cw[1] = sc->op_cw[2] = sc->op_cw[3] = 0; 399 viac3_cbc(&sc->op_cw, sc->op_buf, sc->op_buf, key, 400 crd->crd_len / 16, sc->op_iv); 401 402 if (crp->crp_flags & CRYPTO_F_IMBUF) 403 err = m_copyback((struct mbuf *)crp->crp_buf, 404 crd->crd_skip, crd->crd_len, sc->op_buf, M_NOWAIT); 405 else if (crp->crp_flags & CRYPTO_F_IOV) 406 cuio_copyback((struct uio *)crp->crp_buf, 407 crd->crd_skip, crd->crd_len, sc->op_buf); 408 else 409 memcpy(crp->crp_buf + crd->crd_skip, sc->op_buf, 410 crd->crd_len); 411 412 errout: 413 if (sc->op_buf != NULL) { 414 explicit_bzero(sc->op_buf, crd->crd_len); 415 free(sc->op_buf, M_DEVBUF, crd->crd_len); 416 sc->op_buf = NULL; 417 } 418 419 return (err); 420} 421 422int 423viac3_crypto_process(struct cryptop *crp) 424{ 425 struct viac3_softc *sc = vc3_sc; 426 struct viac3_session *ses; 427 struct cryptodesc *crd; 428 int sesn, err = 0; 429 int i; 430 431 KASSERT(crp->crp_ndesc >= 1); 432 433 sesn = VIAC3_SESSION(crp->crp_sid); 434 if (sesn >= sc->sc_nsessions) { 435 err = EINVAL; 436 goto out; 437 } 438 ses = &sc->sc_sessions[sesn]; 439 if (ses->ses_used == 0) { 440 err = EINVAL; 441 goto out; 442 } 443 444 for (i = 0; i < crp->crp_ndesc; i++) { 445 crd = &crp->crp_desc[i]; 446 switch (crd->crd_alg) { 447 case CRYPTO_AES_CBC: 448 if ((err = viac3_crypto_encdec(crp, crd, ses, sc, 449 crp->crp_buf)) != 0) 450 goto out; 451 break; 452 453 case CRYPTO_MD5_HMAC: 454 case CRYPTO_SHA1_HMAC: 455 case CRYPTO_RIPEMD160_HMAC: 456 case CRYPTO_SHA2_256_HMAC: 457 case CRYPTO_SHA2_384_HMAC: 458 case CRYPTO_SHA2_512_HMAC: 459 if ((err = viac3_crypto_swauth(crp, crd, ses->swd, 460 crp->crp_buf)) != 0) 461 goto out; 462 break; 463 464 default: 465 err = EINVAL; 466 goto out; 467 } 468 } 469out: 470 return (err); 471} 472 473#endif /* CRYPTO */ 474 475/* 476 * Note, the VIA C3 Nehemiah provides 4 internal 8-byte buffers, which 477 * store random data, and can be accessed a lot quicker than waiting 478 * for new data to be generated. As we are using every 8th bit only 479 * due to whitening. Since the RNG generates in excess of 21KB/s at 480 * its worst, collecting 64 bytes worth of entropy should not affect 481 * things significantly. 482 * 483 * Note, due to some weirdness in the RNG, we need at least 7 bytes 484 * extra on the end of our buffer. Also, there is an outside chance 485 * that the VIA RNG can "wedge", as the generated bit-rate is variable. 486 * We could do all sorts of startup testing and things, but 487 * frankly, I don't really see the point. If the RNG wedges, then the 488 * chances of you having a defective CPU are very high. Let it wedge. 489 * 490 * Adding to the whole confusion, in order to access the RNG, we need 491 * to have FXSR support enabled, and the correct FPU enable bits must 492 * be there to enable the FPU in kernel. It would be nice if all this 493 * mumbo-jumbo was not needed in order to use the RNG. Oh well, life 494 * does go on... 495 */ 496#define VIAC3_RNG_BUFSIZ 16 /* 32bit words */ 497struct timeout viac3_rnd_tmo; 498int viac3_rnd_present; 499 500void 501viac3_rnd(void *v) 502{ 503 struct timeout *tmo = v; 504 unsigned int *p, i, rv, creg0, len = VIAC3_RNG_BUFSIZ; 505 static int buffer[VIAC3_RNG_BUFSIZ + 2]; /* XXX why + 2? */ 506#ifdef MULTIPROCESSOR 507 int s = splipi(); 508#endif 509 510 creg0 = rcr0(); /* Permit access to SIMD/FPU path */ 511 lcr0(creg0 & ~(CR0_EM|CR0_TS)); 512 513 /* 514 * Here we collect the random data from the VIA C3 RNG. We make 515 * sure that we turn on maximum whitening (%edx[0,1] == "11"), so 516 * that we get the best random data possible. 517 */ 518 __asm volatile("rep xstorerng" 519 : "=a" (rv) : "d" (3), "D" (buffer), "c" (len*sizeof(int)) 520 : "memory", "cc"); 521 522 lcr0(creg0); 523 524#ifdef MULTIPROCESSOR 525 splx(s); 526#endif 527 528 for (i = 0, p = buffer; i < VIAC3_RNG_BUFSIZ; i++, p++) 529 enqueue_randomness(*p); 530 531 timeout_add_msec(tmo, 10); 532} 533