1/* 2 * vim:sw=4 ts=8 3 */ 4/* 5 * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com> 6 * 7 * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 8 * reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 1. Redistributions of source code must retain the above copyright notice, 13 * this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright notice, 15 * this list of conditions and the following disclaimer in the documentation 16 * and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Cavium Networks 20 * 4. Cavium Networks' name may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * This Software, including technical data, may be subject to U.S. export 24 * control laws, including the U.S. Export Administration Act and its 25 * associated regulations, and may be subject to export or import regulations 26 * in other countries. You warrant that You will comply strictly in all 27 * respects with all such regulations and acknowledge that you have the 28 * responsibility to obtain licenses to export, re-export or import the 29 * Software. 30 * 31 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND 32 * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, 33 * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE 34 * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR 35 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM 36 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, 37 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF 38 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR 39 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR 40 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 41*/ 42/****************************************************************************/ 43 44#include <sys/cdefs.h> 45__FBSDID("$FreeBSD: releng/10.3/sys/mips/cavium/cryptocteon/cavium_crypto.c 229677 2012-01-06 01:23:26Z gonzo $"); 46 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/kernel.h> 50#include <sys/module.h> 51#include <sys/malloc.h> 52#include <sys/uio.h> 53 54#include <opencrypto/cryptodev.h> 55 56#include <contrib/octeon-sdk/cvmx.h> 57 58#include <mips/cavium/cryptocteon/cryptocteonvar.h> 59 60/****************************************************************************/ 61 62#define IOV_INIT(iov, ptr, idx, len) \ 63 do { \ 64 (idx) = 0; \ 65 (ptr) = (iov)[(idx)].iov_base; \ 66 (len) = (iov)[(idx)].iov_len; \ 67 } while (0) 68 69/* 70 * XXX 71 * It would be better if this were an IOV_READ/IOV_WRITE macro instead so 72 * that we could detect overflow before it happens rather than right after, 73 * which is especially bad since there is usually no IOV_CONSUME after the 74 * final read or write. 75 */ 76#define IOV_CONSUME(iov, ptr, idx, len) \ 77 do { \ 78 if ((len) > sizeof *(ptr)) { \ 79 (len) -= sizeof *(ptr); \ 80 (ptr)++; \ 81 } else { \ 82 if ((len) != sizeof *(ptr)) \ 83 panic("%s: went past end of iovec.", __func__); \ 84 (idx)++; \ 85 (ptr) = (iov)[(idx)].iov_base; \ 86 (len) = (iov)[(idx)].iov_len; \ 87 } \ 88 } while (0) 89 90#define ESP_HEADER_LENGTH 8 91#define DES_CBC_IV_LENGTH 8 92#define AES_CBC_IV_LENGTH 16 93#define ESP_HMAC_LEN 12 94 95#define ESP_HEADER_LENGTH 8 96#define DES_CBC_IV_LENGTH 8 97 98/****************************************************************************/ 99 100#define CVM_LOAD_SHA_UNIT(dat, next) { \ 101 if (next == 0) { \ 102 next = 1; \ 103 CVMX_MT_HSH_DAT (dat, 0); \ 104 } else if (next == 1) { \ 105 next = 2; \ 106 CVMX_MT_HSH_DAT (dat, 1); \ 107 } else if (next == 2) { \ 108 next = 3; \ 109 CVMX_MT_HSH_DAT (dat, 2); \ 110 } else if (next == 3) { \ 111 next = 4; \ 112 CVMX_MT_HSH_DAT (dat, 3); \ 113 } else if (next == 4) { \ 114 next = 5; \ 115 CVMX_MT_HSH_DAT (dat, 4); \ 116 } else if (next == 5) { \ 117 next = 6; \ 118 CVMX_MT_HSH_DAT (dat, 5); \ 119 } else if (next == 6) { \ 120 next = 7; \ 121 CVMX_MT_HSH_DAT (dat, 6); \ 122 } else { \ 123 CVMX_MT_HSH_STARTSHA (dat); \ 124 next = 0; \ 125 } \ 126} 127 128#define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \ 129 if (next == 0) { \ 130 CVMX_MT_HSH_DAT (dat1, 0); \ 131 CVMX_MT_HSH_DAT (dat2, 1); \ 132 next = 2; \ 133 } else if (next == 1) { \ 134 CVMX_MT_HSH_DAT (dat1, 1); \ 135 CVMX_MT_HSH_DAT (dat2, 2); \ 136 next = 3; \ 137 } else if (next == 2) { \ 138 CVMX_MT_HSH_DAT (dat1, 2); \ 139 CVMX_MT_HSH_DAT (dat2, 3); \ 140 next = 4; \ 141 } else if (next == 3) { \ 142 CVMX_MT_HSH_DAT (dat1, 3); \ 143 CVMX_MT_HSH_DAT (dat2, 4); \ 144 next = 5; \ 145 } else if (next == 4) { \ 146 CVMX_MT_HSH_DAT (dat1, 4); \ 147 CVMX_MT_HSH_DAT (dat2, 5); \ 148 next = 6; \ 149 } else if (next == 5) { \ 150 CVMX_MT_HSH_DAT (dat1, 5); \ 151 CVMX_MT_HSH_DAT (dat2, 6); \ 152 next = 7; \ 153 } else if (next == 6) { \ 154 CVMX_MT_HSH_DAT (dat1, 6); \ 155 CVMX_MT_HSH_STARTSHA (dat2); \ 156 next = 0; \ 157 } else { \ 158 CVMX_MT_HSH_STARTSHA (dat1); \ 159 CVMX_MT_HSH_DAT (dat2, 0); \ 160 next = 1; \ 161 } \ 162} 163 164/****************************************************************************/ 165 166#define CVM_LOAD_MD5_UNIT(dat, next) { \ 167 if (next == 0) { \ 168 next = 1; \ 169 CVMX_MT_HSH_DAT (dat, 0); \ 170 } else if (next == 1) { \ 171 next = 2; \ 172 CVMX_MT_HSH_DAT (dat, 1); \ 173 } else if (next == 2) { \ 174 next = 3; \ 175 CVMX_MT_HSH_DAT (dat, 2); \ 176 } else if (next == 3) { \ 177 next = 4; \ 178 CVMX_MT_HSH_DAT (dat, 3); \ 179 } else if (next == 4) { \ 180 next = 5; \ 181 CVMX_MT_HSH_DAT (dat, 4); \ 182 } else if (next == 5) { \ 183 next = 6; \ 184 CVMX_MT_HSH_DAT (dat, 5); \ 185 } else if (next == 6) { \ 186 next = 7; \ 187 CVMX_MT_HSH_DAT (dat, 6); \ 188 } else { \ 189 CVMX_MT_HSH_STARTMD5 (dat); \ 190 next = 0; \ 191 } \ 192} 193 194#define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \ 195 if (next == 0) { \ 196 CVMX_MT_HSH_DAT (dat1, 0); \ 197 CVMX_MT_HSH_DAT (dat2, 1); \ 198 next = 2; \ 199 } else if (next == 1) { \ 200 CVMX_MT_HSH_DAT (dat1, 1); \ 201 CVMX_MT_HSH_DAT (dat2, 2); \ 202 next = 3; \ 203 } else if (next == 2) { \ 204 CVMX_MT_HSH_DAT (dat1, 2); \ 205 CVMX_MT_HSH_DAT (dat2, 3); \ 206 next = 4; \ 207 } else if (next == 3) { \ 208 CVMX_MT_HSH_DAT (dat1, 3); \ 209 CVMX_MT_HSH_DAT (dat2, 4); \ 210 next = 5; \ 211 } else if (next == 4) { \ 212 CVMX_MT_HSH_DAT (dat1, 4); \ 213 CVMX_MT_HSH_DAT (dat2, 5); \ 214 next = 6; \ 215 } else if (next == 5) { \ 216 CVMX_MT_HSH_DAT (dat1, 5); \ 217 CVMX_MT_HSH_DAT (dat2, 6); \ 218 next = 7; \ 219 } else if (next == 6) { \ 220 CVMX_MT_HSH_DAT (dat1, 6); \ 221 CVMX_MT_HSH_STARTMD5 (dat2); \ 222 next = 0; \ 223 } else { \ 224 CVMX_MT_HSH_STARTMD5 (dat1); \ 225 CVMX_MT_HSH_DAT (dat2, 0); \ 226 next = 1; \ 227 } \ 228} 229 230/****************************************************************************/ 231 232void 233octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer) 234{ 235 uint8_t hash_key[64]; 236 uint64_t *key1; 237 register uint64_t xor1 = 0x3636363636363636ULL; 238 register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; 239 240 dprintf("%s()\n", __func__); 241 242 memset(hash_key, 0, sizeof(hash_key)); 243 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); 244 key1 = (uint64_t *) hash_key; 245 if (auth) { 246 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); 247 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); 248 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); 249 } else { 250 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); 251 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); 252 } 253 254 CVMX_MT_HSH_DAT((*key1 ^ xor1), 0); 255 key1++; 256 CVMX_MT_HSH_DAT((*key1 ^ xor1), 1); 257 key1++; 258 CVMX_MT_HSH_DAT((*key1 ^ xor1), 2); 259 key1++; 260 CVMX_MT_HSH_DAT((*key1 ^ xor1), 3); 261 key1++; 262 CVMX_MT_HSH_DAT((*key1 ^ xor1), 4); 263 key1++; 264 CVMX_MT_HSH_DAT((*key1 ^ xor1), 5); 265 key1++; 266 CVMX_MT_HSH_DAT((*key1 ^ xor1), 6); 267 key1++; 268 if (auth) 269 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1)); 270 else 271 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1)); 272 273 CVMX_MF_HSH_IV(inner[0], 0); 274 CVMX_MF_HSH_IV(inner[1], 1); 275 if (auth) { 276 inner[2] = 0; 277 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2); 278 } 279 280 memset(hash_key, 0, sizeof(hash_key)); 281 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); 282 key1 = (uint64_t *) hash_key; 283 if (auth) { 284 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); 285 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); 286 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); 287 } else { 288 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); 289 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); 290 } 291 292 CVMX_MT_HSH_DAT((*key1 ^ xor2), 0); 293 key1++; 294 CVMX_MT_HSH_DAT((*key1 ^ xor2), 1); 295 key1++; 296 CVMX_MT_HSH_DAT((*key1 ^ xor2), 2); 297 key1++; 298 CVMX_MT_HSH_DAT((*key1 ^ xor2), 3); 299 key1++; 300 CVMX_MT_HSH_DAT((*key1 ^ xor2), 4); 301 key1++; 302 CVMX_MT_HSH_DAT((*key1 ^ xor2), 5); 303 key1++; 304 CVMX_MT_HSH_DAT((*key1 ^ xor2), 6); 305 key1++; 306 if (auth) 307 CVMX_MT_HSH_STARTSHA((*key1 ^ xor2)); 308 else 309 CVMX_MT_HSH_STARTMD5((*key1 ^ xor2)); 310 311 CVMX_MF_HSH_IV(outer[0], 0); 312 CVMX_MF_HSH_IV(outer[1], 1); 313 if (auth) { 314 outer[2] = 0; 315 CVMX_MF_HSH_IV(outer[2], 2); 316 } 317 return; 318} 319 320/****************************************************************************/ 321/* DES functions */ 322 323int 324octo_des_cbc_encrypt( 325 struct octo_sess *od, 326 struct iovec *iov, size_t iovcnt, size_t iovlen, 327 int auth_off, int auth_len, 328 int crypt_off, int crypt_len, 329 int icv_off, uint8_t *ivp) 330{ 331 uint64_t *data; 332 int data_i, data_l; 333 334 dprintf("%s()\n", __func__); 335 336 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 337 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 338 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 339 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 340 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 341 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 342 return -EINVAL; 343 } 344 345 IOV_INIT(iov, data, data_i, data_l); 346 347 CVMX_PREFETCH0(ivp); 348 CVMX_PREFETCH0(od->octo_enckey); 349 350 351 /* load 3DES Key */ 352 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 353 if (od->octo_encklen == 24) { 354 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 355 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 356 } else if (od->octo_encklen == 8) { 357 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 358 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 359 } else { 360 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 361 return -EINVAL; 362 } 363 364 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 365 366 while (crypt_off > 0) { 367 IOV_CONSUME(iov, data, data_i, data_l); 368 crypt_off -= 8; 369 } 370 371 while (crypt_len > 0) { 372 CVMX_MT_3DES_ENC_CBC(*data); 373 CVMX_MF_3DES_RESULT(*data); 374 IOV_CONSUME(iov, data, data_i, data_l); 375 crypt_len -= 8; 376 } 377 378 return 0; 379} 380 381 382int 383octo_des_cbc_decrypt( 384 struct octo_sess *od, 385 struct iovec *iov, size_t iovcnt, size_t iovlen, 386 int auth_off, int auth_len, 387 int crypt_off, int crypt_len, 388 int icv_off, uint8_t *ivp) 389{ 390 uint64_t *data; 391 int data_i, data_l; 392 393 dprintf("%s()\n", __func__); 394 395 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 396 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 397 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 398 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 399 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 400 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 401 return -EINVAL; 402 } 403 404 IOV_INIT(iov, data, data_i, data_l); 405 406 CVMX_PREFETCH0(ivp); 407 CVMX_PREFETCH0(od->octo_enckey); 408 409 /* load 3DES Key */ 410 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 411 if (od->octo_encklen == 24) { 412 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 413 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 414 } else if (od->octo_encklen == 8) { 415 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 416 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 417 } else { 418 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 419 return -EINVAL; 420 } 421 422 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 423 424 while (crypt_off > 0) { 425 IOV_CONSUME(iov, data, data_i, data_l); 426 crypt_off -= 8; 427 } 428 429 while (crypt_len > 0) { 430 CVMX_MT_3DES_DEC_CBC(*data); 431 CVMX_MF_3DES_RESULT(*data); 432 IOV_CONSUME(iov, data, data_i, data_l); 433 crypt_len -= 8; 434 } 435 436 return 0; 437} 438 439/****************************************************************************/ 440/* AES functions */ 441 442int 443octo_aes_cbc_encrypt( 444 struct octo_sess *od, 445 struct iovec *iov, size_t iovcnt, size_t iovlen, 446 int auth_off, int auth_len, 447 int crypt_off, int crypt_len, 448 int icv_off, uint8_t *ivp) 449{ 450 uint64_t *data, *pdata; 451 int data_i, data_l; 452 453 dprintf("%s()\n", __func__); 454 455 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 456 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 457 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 458 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 459 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 460 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 461 return -EINVAL; 462 } 463 464 IOV_INIT(iov, data, data_i, data_l); 465 466 CVMX_PREFETCH0(ivp); 467 CVMX_PREFETCH0(od->octo_enckey); 468 469 /* load AES Key */ 470 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 471 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 472 473 if (od->octo_encklen == 16) { 474 CVMX_MT_AES_KEY(0x0, 2); 475 CVMX_MT_AES_KEY(0x0, 3); 476 } else if (od->octo_encklen == 24) { 477 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 478 CVMX_MT_AES_KEY(0x0, 3); 479 } else if (od->octo_encklen == 32) { 480 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 481 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 482 } else { 483 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 484 return -EINVAL; 485 } 486 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 487 488 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 489 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 490 491 while (crypt_off > 0) { 492 IOV_CONSUME(iov, data, data_i, data_l); 493 crypt_off -= 8; 494 } 495 496 while (crypt_len > 0) { 497 pdata = data; 498 CVMX_MT_AES_ENC_CBC0(*data); 499 IOV_CONSUME(iov, data, data_i, data_l); 500 CVMX_MT_AES_ENC_CBC1(*data); 501 CVMX_MF_AES_RESULT(*pdata, 0); 502 CVMX_MF_AES_RESULT(*data, 1); 503 IOV_CONSUME(iov, data, data_i, data_l); 504 crypt_len -= 16; 505 } 506 507 return 0; 508} 509 510 511int 512octo_aes_cbc_decrypt( 513 struct octo_sess *od, 514 struct iovec *iov, size_t iovcnt, size_t iovlen, 515 int auth_off, int auth_len, 516 int crypt_off, int crypt_len, 517 int icv_off, uint8_t *ivp) 518{ 519 uint64_t *data, *pdata; 520 int data_i, data_l; 521 522 dprintf("%s()\n", __func__); 523 524 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 525 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { 526 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 527 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 528 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 529 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 530 return -EINVAL; 531 } 532 533 IOV_INIT(iov, data, data_i, data_l); 534 535 CVMX_PREFETCH0(ivp); 536 CVMX_PREFETCH0(od->octo_enckey); 537 538 /* load AES Key */ 539 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 540 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 541 542 if (od->octo_encklen == 16) { 543 CVMX_MT_AES_KEY(0x0, 2); 544 CVMX_MT_AES_KEY(0x0, 3); 545 } else if (od->octo_encklen == 24) { 546 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 547 CVMX_MT_AES_KEY(0x0, 3); 548 } else if (od->octo_encklen == 32) { 549 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 550 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 551 } else { 552 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 553 return -EINVAL; 554 } 555 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 556 557 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 558 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 559 560 while (crypt_off > 0) { 561 IOV_CONSUME(iov, data, data_i, data_l); 562 crypt_off -= 8; 563 } 564 565 while (crypt_len > 0) { 566 pdata = data; 567 CVMX_MT_AES_DEC_CBC0(*data); 568 IOV_CONSUME(iov, data, data_i, data_l); 569 CVMX_MT_AES_DEC_CBC1(*data); 570 CVMX_MF_AES_RESULT(*pdata, 0); 571 CVMX_MF_AES_RESULT(*data, 1); 572 IOV_CONSUME(iov, data, data_i, data_l); 573 crypt_len -= 16; 574 } 575 576 return 0; 577} 578 579/****************************************************************************/ 580/* MD5 */ 581 582int 583octo_null_md5_encrypt( 584 struct octo_sess *od, 585 struct iovec *iov, size_t iovcnt, size_t iovlen, 586 int auth_off, int auth_len, 587 int crypt_off, int crypt_len, 588 int icv_off, uint8_t *ivp) 589{ 590 register int next = 0; 591 uint64_t *data; 592 uint64_t tmp1, tmp2; 593 int data_i, data_l, alen = auth_len; 594 595 dprintf("%s()\n", __func__); 596 597 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || 598 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { 599 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 600 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 601 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 602 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 603 return -EINVAL; 604 } 605 606 IOV_INIT(iov, data, data_i, data_l); 607 608 /* Load MD5 IV */ 609 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 610 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 611 612 while (auth_off > 0) { 613 IOV_CONSUME(iov, data, data_i, data_l); 614 auth_off -= 8; 615 } 616 617 while (auth_len > 0) { 618 CVM_LOAD_MD5_UNIT(*data, next); 619 auth_len -= 8; 620 IOV_CONSUME(iov, data, data_i, data_l); 621 } 622 623 /* finish the hash */ 624 CVMX_PREFETCH0(od->octo_hmouter); 625#if 0 626 if (__predict_false(inplen)) { 627 uint64_t tmp = 0; 628 uint8_t *p = (uint8_t *) & tmp; 629 p[inplen] = 0x80; 630 do { 631 inplen--; 632 p[inplen] = ((uint8_t *) data)[inplen]; 633 } while (inplen); 634 CVM_LOAD_MD5_UNIT(tmp, next); 635 } else { 636 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 637 } 638#else 639 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 640#endif 641 642 /* Finish Inner hash */ 643 while (next != 7) { 644 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 645 } 646 CVMX_ES64(tmp1, ((alen + 64) << 3)); 647 CVM_LOAD_MD5_UNIT(tmp1, next); 648 649 /* Get the inner hash of HMAC */ 650 CVMX_MF_HSH_IV(tmp1, 0); 651 CVMX_MF_HSH_IV(tmp2, 1); 652 653 /* Initialize hash unit */ 654 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 655 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 656 657 CVMX_MT_HSH_DAT(tmp1, 0); 658 CVMX_MT_HSH_DAT(tmp2, 1); 659 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 660 CVMX_MT_HSH_DATZ(3); 661 CVMX_MT_HSH_DATZ(4); 662 CVMX_MT_HSH_DATZ(5); 663 CVMX_MT_HSH_DATZ(6); 664 CVMX_ES64(tmp1, ((64 + 16) << 3)); 665 CVMX_MT_HSH_STARTMD5(tmp1); 666 667 /* save the HMAC */ 668 IOV_INIT(iov, data, data_i, data_l); 669 while (icv_off > 0) { 670 IOV_CONSUME(iov, data, data_i, data_l); 671 icv_off -= 8; 672 } 673 CVMX_MF_HSH_IV(*data, 0); 674 IOV_CONSUME(iov, data, data_i, data_l); 675 CVMX_MF_HSH_IV(tmp1, 1); 676 *(uint32_t *)data = (uint32_t) (tmp1 >> 32); 677 678 return 0; 679} 680 681/****************************************************************************/ 682/* SHA1 */ 683 684int 685octo_null_sha1_encrypt( 686 struct octo_sess *od, 687 struct iovec *iov, size_t iovcnt, size_t iovlen, 688 int auth_off, int auth_len, 689 int crypt_off, int crypt_len, 690 int icv_off, uint8_t *ivp) 691{ 692 register int next = 0; 693 uint64_t *data; 694 uint64_t tmp1, tmp2, tmp3; 695 int data_i, data_l, alen = auth_len; 696 697 dprintf("%s()\n", __func__); 698 699 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || 700 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { 701 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 702 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 703 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 704 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 705 return -EINVAL; 706 } 707 708 IOV_INIT(iov, data, data_i, data_l); 709 710 /* Load SHA1 IV */ 711 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 712 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 713 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 714 715 while (auth_off > 0) { 716 IOV_CONSUME(iov, data, data_i, data_l); 717 auth_off -= 8; 718 } 719 720 while (auth_len > 0) { 721 CVM_LOAD_SHA_UNIT(*data, next); 722 auth_len -= 8; 723 IOV_CONSUME(iov, data, data_i, data_l); 724 } 725 726 /* finish the hash */ 727 CVMX_PREFETCH0(od->octo_hmouter); 728#if 0 729 if (__predict_false(inplen)) { 730 uint64_t tmp = 0; 731 uint8_t *p = (uint8_t *) & tmp; 732 p[inplen] = 0x80; 733 do { 734 inplen--; 735 p[inplen] = ((uint8_t *) data)[inplen]; 736 } while (inplen); 737 CVM_LOAD_MD5_UNIT(tmp, next); 738 } else { 739 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 740 } 741#else 742 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 743#endif 744 745 /* Finish Inner hash */ 746 while (next != 7) { 747 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 748 } 749 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 750 751 /* Get the inner hash of HMAC */ 752 CVMX_MF_HSH_IV(tmp1, 0); 753 CVMX_MF_HSH_IV(tmp2, 1); 754 tmp3 = 0; 755 CVMX_MF_HSH_IV(tmp3, 2); 756 757 /* Initialize hash unit */ 758 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 759 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 760 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 761 762 CVMX_MT_HSH_DAT(tmp1, 0); 763 CVMX_MT_HSH_DAT(tmp2, 1); 764 tmp3 |= 0x0000000080000000; 765 CVMX_MT_HSH_DAT(tmp3, 2); 766 CVMX_MT_HSH_DATZ(3); 767 CVMX_MT_HSH_DATZ(4); 768 CVMX_MT_HSH_DATZ(5); 769 CVMX_MT_HSH_DATZ(6); 770 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 771 772 /* save the HMAC */ 773 IOV_INIT(iov, data, data_i, data_l); 774 while (icv_off > 0) { 775 IOV_CONSUME(iov, data, data_i, data_l); 776 icv_off -= 8; 777 } 778 CVMX_MF_HSH_IV(*data, 0); 779 IOV_CONSUME(iov, data, data_i, data_l); 780 CVMX_MF_HSH_IV(tmp1, 1); 781 *(uint32_t *)data = (uint32_t) (tmp1 >> 32); 782 783 return 0; 784} 785 786/****************************************************************************/ 787/* DES MD5 */ 788 789int 790octo_des_cbc_md5_encrypt( 791 struct octo_sess *od, 792 struct iovec *iov, size_t iovcnt, size_t iovlen, 793 int auth_off, int auth_len, 794 int crypt_off, int crypt_len, 795 int icv_off, uint8_t *ivp) 796{ 797 register int next = 0; 798 union { 799 uint32_t data32[2]; 800 uint64_t data64[1]; 801 } mydata; 802 uint64_t *data = &mydata.data64[0]; 803 uint32_t *data32; 804 uint64_t tmp1, tmp2; 805 int data_i, data_l, alen = auth_len; 806 807 dprintf("%s()\n", __func__); 808 809 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 810 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 811 (crypt_len & 0x7) || 812 (auth_len & 0x7) || 813 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 814 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 815 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 816 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 817 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 818 return -EINVAL; 819 } 820 821 IOV_INIT(iov, data32, data_i, data_l); 822 823 CVMX_PREFETCH0(ivp); 824 CVMX_PREFETCH0(od->octo_enckey); 825 826 /* load 3DES Key */ 827 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 828 if (od->octo_encklen == 24) { 829 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 830 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 831 } else if (od->octo_encklen == 8) { 832 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 833 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 834 } else { 835 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 836 return -EINVAL; 837 } 838 839 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 840 841 /* Load MD5 IV */ 842 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 843 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 844 845 while (crypt_off > 0 && auth_off > 0) { 846 IOV_CONSUME(iov, data32, data_i, data_l); 847 crypt_off -= 4; 848 auth_off -= 4; 849 } 850 851 while (crypt_len > 0 || auth_len > 0) { 852 uint32_t *first = data32; 853 mydata.data32[0] = *first; 854 IOV_CONSUME(iov, data32, data_i, data_l); 855 mydata.data32[1] = *data32; 856 if (crypt_off <= 0) { 857 if (crypt_len > 0) { 858 CVMX_MT_3DES_ENC_CBC(*data); 859 CVMX_MF_3DES_RESULT(*data); 860 crypt_len -= 8; 861 } 862 } else 863 crypt_off -= 8; 864 if (auth_off <= 0) { 865 if (auth_len > 0) { 866 CVM_LOAD_MD5_UNIT(*data, next); 867 auth_len -= 8; 868 } 869 } else 870 auth_off -= 8; 871 *first = mydata.data32[0]; 872 *data32 = mydata.data32[1]; 873 IOV_CONSUME(iov, data32, data_i, data_l); 874 } 875 876 /* finish the hash */ 877 CVMX_PREFETCH0(od->octo_hmouter); 878#if 0 879 if (__predict_false(inplen)) { 880 uint64_t tmp = 0; 881 uint8_t *p = (uint8_t *) & tmp; 882 p[inplen] = 0x80; 883 do { 884 inplen--; 885 p[inplen] = ((uint8_t *) data)[inplen]; 886 } while (inplen); 887 CVM_LOAD_MD5_UNIT(tmp, next); 888 } else { 889 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 890 } 891#else 892 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 893#endif 894 895 /* Finish Inner hash */ 896 while (next != 7) { 897 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 898 } 899 CVMX_ES64(tmp1, ((alen + 64) << 3)); 900 CVM_LOAD_MD5_UNIT(tmp1, next); 901 902 /* Get the inner hash of HMAC */ 903 CVMX_MF_HSH_IV(tmp1, 0); 904 CVMX_MF_HSH_IV(tmp2, 1); 905 906 /* Initialize hash unit */ 907 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 908 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 909 910 CVMX_MT_HSH_DAT(tmp1, 0); 911 CVMX_MT_HSH_DAT(tmp2, 1); 912 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 913 CVMX_MT_HSH_DATZ(3); 914 CVMX_MT_HSH_DATZ(4); 915 CVMX_MT_HSH_DATZ(5); 916 CVMX_MT_HSH_DATZ(6); 917 CVMX_ES64(tmp1, ((64 + 16) << 3)); 918 CVMX_MT_HSH_STARTMD5(tmp1); 919 920 /* save the HMAC */ 921 IOV_INIT(iov, data32, data_i, data_l); 922 while (icv_off > 0) { 923 IOV_CONSUME(iov, data32, data_i, data_l); 924 icv_off -= 4; 925 } 926 CVMX_MF_HSH_IV(tmp1, 0); 927 *data32 = (uint32_t) (tmp1 >> 32); 928 IOV_CONSUME(iov, data32, data_i, data_l); 929 *data32 = (uint32_t) tmp1; 930 IOV_CONSUME(iov, data32, data_i, data_l); 931 CVMX_MF_HSH_IV(tmp1, 1); 932 *data32 = (uint32_t) (tmp1 >> 32); 933 934 return 0; 935} 936 937int 938octo_des_cbc_md5_decrypt( 939 struct octo_sess *od, 940 struct iovec *iov, size_t iovcnt, size_t iovlen, 941 int auth_off, int auth_len, 942 int crypt_off, int crypt_len, 943 int icv_off, uint8_t *ivp) 944{ 945 register int next = 0; 946 union { 947 uint32_t data32[2]; 948 uint64_t data64[1]; 949 } mydata; 950 uint64_t *data = &mydata.data64[0]; 951 uint32_t *data32; 952 uint64_t tmp1, tmp2; 953 int data_i, data_l, alen = auth_len; 954 955 dprintf("%s()\n", __func__); 956 957 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 958 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 959 (crypt_len & 0x7) || 960 (auth_len & 0x7) || 961 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 962 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 963 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 964 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 965 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 966 return -EINVAL; 967 } 968 969 IOV_INIT(iov, data32, data_i, data_l); 970 971 CVMX_PREFETCH0(ivp); 972 CVMX_PREFETCH0(od->octo_enckey); 973 974 /* load 3DES Key */ 975 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 976 if (od->octo_encklen == 24) { 977 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 978 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 979 } else if (od->octo_encklen == 8) { 980 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 981 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 982 } else { 983 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 984 return -EINVAL; 985 } 986 987 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 988 989 /* Load MD5 IV */ 990 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 991 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 992 993 while (crypt_off > 0 && auth_off > 0) { 994 IOV_CONSUME(iov, data32, data_i, data_l); 995 crypt_off -= 4; 996 auth_off -= 4; 997 } 998 999 while (crypt_len > 0 || auth_len > 0) { 1000 uint32_t *first = data32; 1001 mydata.data32[0] = *first; 1002 IOV_CONSUME(iov, data32, data_i, data_l); 1003 mydata.data32[1] = *data32; 1004 if (auth_off <= 0) { 1005 if (auth_len > 0) { 1006 CVM_LOAD_MD5_UNIT(*data, next); 1007 auth_len -= 8; 1008 } 1009 } else 1010 auth_off -= 8; 1011 if (crypt_off <= 0) { 1012 if (crypt_len > 0) { 1013 CVMX_MT_3DES_DEC_CBC(*data); 1014 CVMX_MF_3DES_RESULT(*data); 1015 crypt_len -= 8; 1016 } 1017 } else 1018 crypt_off -= 8; 1019 *first = mydata.data32[0]; 1020 *data32 = mydata.data32[1]; 1021 IOV_CONSUME(iov, data32, data_i, data_l); 1022 } 1023 1024 /* finish the hash */ 1025 CVMX_PREFETCH0(od->octo_hmouter); 1026#if 0 1027 if (__predict_false(inplen)) { 1028 uint64_t tmp = 0; 1029 uint8_t *p = (uint8_t *) & tmp; 1030 p[inplen] = 0x80; 1031 do { 1032 inplen--; 1033 p[inplen] = ((uint8_t *) data)[inplen]; 1034 } while (inplen); 1035 CVM_LOAD_MD5_UNIT(tmp, next); 1036 } else { 1037 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1038 } 1039#else 1040 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1041#endif 1042 1043 /* Finish Inner hash */ 1044 while (next != 7) { 1045 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 1046 } 1047 CVMX_ES64(tmp1, ((alen + 64) << 3)); 1048 CVM_LOAD_MD5_UNIT(tmp1, next); 1049 1050 /* Get the inner hash of HMAC */ 1051 CVMX_MF_HSH_IV(tmp1, 0); 1052 CVMX_MF_HSH_IV(tmp2, 1); 1053 1054 /* Initialize hash unit */ 1055 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1056 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1057 1058 CVMX_MT_HSH_DAT(tmp1, 0); 1059 CVMX_MT_HSH_DAT(tmp2, 1); 1060 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 1061 CVMX_MT_HSH_DATZ(3); 1062 CVMX_MT_HSH_DATZ(4); 1063 CVMX_MT_HSH_DATZ(5); 1064 CVMX_MT_HSH_DATZ(6); 1065 CVMX_ES64(tmp1, ((64 + 16) << 3)); 1066 CVMX_MT_HSH_STARTMD5(tmp1); 1067 1068 /* save the HMAC */ 1069 IOV_INIT(iov, data32, data_i, data_l); 1070 while (icv_off > 0) { 1071 IOV_CONSUME(iov, data32, data_i, data_l); 1072 icv_off -= 4; 1073 } 1074 CVMX_MF_HSH_IV(tmp1, 0); 1075 *data32 = (uint32_t) (tmp1 >> 32); 1076 IOV_CONSUME(iov, data32, data_i, data_l); 1077 *data32 = (uint32_t) tmp1; 1078 IOV_CONSUME(iov, data32, data_i, data_l); 1079 CVMX_MF_HSH_IV(tmp1, 1); 1080 *data32 = (uint32_t) (tmp1 >> 32); 1081 1082 return 0; 1083} 1084 1085/****************************************************************************/ 1086/* DES SHA */ 1087 1088int 1089octo_des_cbc_sha1_encrypt( 1090 struct octo_sess *od, 1091 struct iovec *iov, size_t iovcnt, size_t iovlen, 1092 int auth_off, int auth_len, 1093 int crypt_off, int crypt_len, 1094 int icv_off, uint8_t *ivp) 1095{ 1096 register int next = 0; 1097 union { 1098 uint32_t data32[2]; 1099 uint64_t data64[1]; 1100 } mydata; 1101 uint64_t *data = &mydata.data64[0]; 1102 uint32_t *data32; 1103 uint64_t tmp1, tmp2, tmp3; 1104 int data_i, data_l, alen = auth_len; 1105 1106 dprintf("%s()\n", __func__); 1107 1108 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1109 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1110 (crypt_len & 0x7) || 1111 (auth_len & 0x7) || 1112 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1113 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1114 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1115 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1116 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1117 return -EINVAL; 1118 } 1119 1120 IOV_INIT(iov, data32, data_i, data_l); 1121 1122 CVMX_PREFETCH0(ivp); 1123 CVMX_PREFETCH0(od->octo_enckey); 1124 1125 /* load 3DES Key */ 1126 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1127 if (od->octo_encklen == 24) { 1128 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1129 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1130 } else if (od->octo_encklen == 8) { 1131 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 1132 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 1133 } else { 1134 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1135 return -EINVAL; 1136 } 1137 1138 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 1139 1140 /* Load SHA1 IV */ 1141 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1142 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1143 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 1144 1145 while (crypt_off > 0 && auth_off > 0) { 1146 IOV_CONSUME(iov, data32, data_i, data_l); 1147 crypt_off -= 4; 1148 auth_off -= 4; 1149 } 1150 1151 while (crypt_len > 0 || auth_len > 0) { 1152 uint32_t *first = data32; 1153 mydata.data32[0] = *first; 1154 IOV_CONSUME(iov, data32, data_i, data_l); 1155 mydata.data32[1] = *data32; 1156 if (crypt_off <= 0) { 1157 if (crypt_len > 0) { 1158 CVMX_MT_3DES_ENC_CBC(*data); 1159 CVMX_MF_3DES_RESULT(*data); 1160 crypt_len -= 8; 1161 } 1162 } else 1163 crypt_off -= 8; 1164 if (auth_off <= 0) { 1165 if (auth_len > 0) { 1166 CVM_LOAD_SHA_UNIT(*data, next); 1167 auth_len -= 8; 1168 } 1169 } else 1170 auth_off -= 8; 1171 *first = mydata.data32[0]; 1172 *data32 = mydata.data32[1]; 1173 IOV_CONSUME(iov, data32, data_i, data_l); 1174 } 1175 1176 /* finish the hash */ 1177 CVMX_PREFETCH0(od->octo_hmouter); 1178#if 0 1179 if (__predict_false(inplen)) { 1180 uint64_t tmp = 0; 1181 uint8_t *p = (uint8_t *) & tmp; 1182 p[inplen] = 0x80; 1183 do { 1184 inplen--; 1185 p[inplen] = ((uint8_t *) data)[inplen]; 1186 } while (inplen); 1187 CVM_LOAD_SHA_UNIT(tmp, next); 1188 } else { 1189 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1190 } 1191#else 1192 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1193#endif 1194 1195 /* Finish Inner hash */ 1196 while (next != 7) { 1197 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 1198 } 1199 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 1200 1201 /* Get the inner hash of HMAC */ 1202 CVMX_MF_HSH_IV(tmp1, 0); 1203 CVMX_MF_HSH_IV(tmp2, 1); 1204 tmp3 = 0; 1205 CVMX_MF_HSH_IV(tmp3, 2); 1206 1207 /* Initialize hash unit */ 1208 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1209 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1210 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 1211 1212 CVMX_MT_HSH_DAT(tmp1, 0); 1213 CVMX_MT_HSH_DAT(tmp2, 1); 1214 tmp3 |= 0x0000000080000000; 1215 CVMX_MT_HSH_DAT(tmp3, 2); 1216 CVMX_MT_HSH_DATZ(3); 1217 CVMX_MT_HSH_DATZ(4); 1218 CVMX_MT_HSH_DATZ(5); 1219 CVMX_MT_HSH_DATZ(6); 1220 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 1221 1222 /* save the HMAC */ 1223 IOV_INIT(iov, data32, data_i, data_l); 1224 while (icv_off > 0) { 1225 IOV_CONSUME(iov, data32, data_i, data_l); 1226 icv_off -= 4; 1227 } 1228 CVMX_MF_HSH_IV(tmp1, 0); 1229 *data32 = (uint32_t) (tmp1 >> 32); 1230 IOV_CONSUME(iov, data32, data_i, data_l); 1231 *data32 = (uint32_t) tmp1; 1232 IOV_CONSUME(iov, data32, data_i, data_l); 1233 CVMX_MF_HSH_IV(tmp1, 1); 1234 *data32 = (uint32_t) (tmp1 >> 32); 1235 1236 return 0; 1237} 1238 1239int 1240octo_des_cbc_sha1_decrypt( 1241 struct octo_sess *od, 1242 struct iovec *iov, size_t iovcnt, size_t iovlen, 1243 int auth_off, int auth_len, 1244 int crypt_off, int crypt_len, 1245 int icv_off, uint8_t *ivp) 1246{ 1247 register int next = 0; 1248 union { 1249 uint32_t data32[2]; 1250 uint64_t data64[1]; 1251 } mydata; 1252 uint64_t *data = &mydata.data64[0]; 1253 uint32_t *data32; 1254 uint64_t tmp1, tmp2, tmp3; 1255 int data_i, data_l, alen = auth_len; 1256 1257 dprintf("%s()\n", __func__); 1258 1259 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1260 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1261 (crypt_len & 0x7) || 1262 (auth_len & 0x7) || 1263 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1264 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1265 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1266 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1267 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1268 return -EINVAL; 1269 } 1270 1271 IOV_INIT(iov, data32, data_i, data_l); 1272 1273 CVMX_PREFETCH0(ivp); 1274 CVMX_PREFETCH0(od->octo_enckey); 1275 1276 /* load 3DES Key */ 1277 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1278 if (od->octo_encklen == 24) { 1279 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1280 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1281 } else if (od->octo_encklen == 8) { 1282 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); 1283 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); 1284 } else { 1285 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1286 return -EINVAL; 1287 } 1288 1289 CVMX_MT_3DES_IV(* (uint64_t *) ivp); 1290 1291 /* Load SHA1 IV */ 1292 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1293 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1294 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 1295 1296 while (crypt_off > 0 && auth_off > 0) { 1297 IOV_CONSUME(iov, data32, data_i, data_l); 1298 crypt_off -= 4; 1299 auth_off -= 4; 1300 } 1301 1302 while (crypt_len > 0 || auth_len > 0) { 1303 uint32_t *first = data32; 1304 mydata.data32[0] = *first; 1305 IOV_CONSUME(iov, data32, data_i, data_l); 1306 mydata.data32[1] = *data32; 1307 if (auth_off <= 0) { 1308 if (auth_len > 0) { 1309 CVM_LOAD_SHA_UNIT(*data, next); 1310 auth_len -= 8; 1311 } 1312 } else 1313 auth_off -= 8; 1314 if (crypt_off <= 0) { 1315 if (crypt_len > 0) { 1316 CVMX_MT_3DES_DEC_CBC(*data); 1317 CVMX_MF_3DES_RESULT(*data); 1318 crypt_len -= 8; 1319 } 1320 } else 1321 crypt_off -= 8; 1322 *first = mydata.data32[0]; 1323 *data32 = mydata.data32[1]; 1324 IOV_CONSUME(iov, data32, data_i, data_l); 1325 } 1326 1327 /* finish the hash */ 1328 CVMX_PREFETCH0(od->octo_hmouter); 1329#if 0 1330 if (__predict_false(inplen)) { 1331 uint64_t tmp = 0; 1332 uint8_t *p = (uint8_t *) & tmp; 1333 p[inplen] = 0x80; 1334 do { 1335 inplen--; 1336 p[inplen] = ((uint8_t *) data)[inplen]; 1337 } while (inplen); 1338 CVM_LOAD_SHA_UNIT(tmp, next); 1339 } else { 1340 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1341 } 1342#else 1343 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1344#endif 1345 1346 /* Finish Inner hash */ 1347 while (next != 7) { 1348 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 1349 } 1350 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 1351 1352 /* Get the inner hash of HMAC */ 1353 CVMX_MF_HSH_IV(tmp1, 0); 1354 CVMX_MF_HSH_IV(tmp2, 1); 1355 tmp3 = 0; 1356 CVMX_MF_HSH_IV(tmp3, 2); 1357 1358 /* Initialize hash unit */ 1359 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1360 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1361 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 1362 1363 CVMX_MT_HSH_DAT(tmp1, 0); 1364 CVMX_MT_HSH_DAT(tmp2, 1); 1365 tmp3 |= 0x0000000080000000; 1366 CVMX_MT_HSH_DAT(tmp3, 2); 1367 CVMX_MT_HSH_DATZ(3); 1368 CVMX_MT_HSH_DATZ(4); 1369 CVMX_MT_HSH_DATZ(5); 1370 CVMX_MT_HSH_DATZ(6); 1371 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 1372 /* save the HMAC */ 1373 IOV_INIT(iov, data32, data_i, data_l); 1374 while (icv_off > 0) { 1375 IOV_CONSUME(iov, data32, data_i, data_l); 1376 icv_off -= 4; 1377 } 1378 CVMX_MF_HSH_IV(tmp1, 0); 1379 *data32 = (uint32_t) (tmp1 >> 32); 1380 IOV_CONSUME(iov, data32, data_i, data_l); 1381 *data32 = (uint32_t) tmp1; 1382 IOV_CONSUME(iov, data32, data_i, data_l); 1383 CVMX_MF_HSH_IV(tmp1, 1); 1384 *data32 = (uint32_t) (tmp1 >> 32); 1385 1386 return 0; 1387} 1388 1389/****************************************************************************/ 1390/* AES MD5 */ 1391 1392int 1393octo_aes_cbc_md5_encrypt( 1394 struct octo_sess *od, 1395 struct iovec *iov, size_t iovcnt, size_t iovlen, 1396 int auth_off, int auth_len, 1397 int crypt_off, int crypt_len, 1398 int icv_off, uint8_t *ivp) 1399{ 1400 register int next = 0; 1401 union { 1402 uint32_t data32[2]; 1403 uint64_t data64[1]; 1404 } mydata[2]; 1405 uint64_t *pdata = &mydata[0].data64[0]; 1406 uint64_t *data = &mydata[1].data64[0]; 1407 uint32_t *data32; 1408 uint64_t tmp1, tmp2; 1409 int data_i, data_l, alen = auth_len; 1410 1411 dprintf("%s()\n", __func__); 1412 1413 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1414 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1415 (crypt_len & 0x7) || 1416 (auth_len & 0x7) || 1417 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1418 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1419 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1420 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1421 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1422 return -EINVAL; 1423 } 1424 1425 IOV_INIT(iov, data32, data_i, data_l); 1426 1427 CVMX_PREFETCH0(ivp); 1428 CVMX_PREFETCH0(od->octo_enckey); 1429 1430 /* load AES Key */ 1431 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1432 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1433 1434 if (od->octo_encklen == 16) { 1435 CVMX_MT_AES_KEY(0x0, 2); 1436 CVMX_MT_AES_KEY(0x0, 3); 1437 } else if (od->octo_encklen == 24) { 1438 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1439 CVMX_MT_AES_KEY(0x0, 3); 1440 } else if (od->octo_encklen == 32) { 1441 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1442 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1443 } else { 1444 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1445 return -EINVAL; 1446 } 1447 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1448 1449 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1450 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 1451 1452 /* Load MD5 IV */ 1453 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1454 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1455 1456 while (crypt_off > 0 && auth_off > 0) { 1457 IOV_CONSUME(iov, data32, data_i, data_l); 1458 crypt_off -= 4; 1459 auth_off -= 4; 1460 } 1461 1462 while (crypt_len > 0 || auth_len > 0) { 1463 uint32_t *pdata32[3]; 1464 1465 pdata32[0] = data32; 1466 mydata[0].data32[0] = *data32; 1467 IOV_CONSUME(iov, data32, data_i, data_l); 1468 1469 pdata32[1] = data32; 1470 mydata[0].data32[1] = *data32; 1471 IOV_CONSUME(iov, data32, data_i, data_l); 1472 1473 pdata32[2] = data32; 1474 mydata[1].data32[0] = *data32; 1475 IOV_CONSUME(iov, data32, data_i, data_l); 1476 1477 mydata[1].data32[1] = *data32; 1478 1479 1480 if (crypt_off <= 0) { 1481 if (crypt_len > 0) { 1482 CVMX_MT_AES_ENC_CBC0(*pdata); 1483 CVMX_MT_AES_ENC_CBC1(*data); 1484 CVMX_MF_AES_RESULT(*pdata, 0); 1485 CVMX_MF_AES_RESULT(*data, 1); 1486 crypt_len -= 16; 1487 } 1488 } else 1489 crypt_off -= 16; 1490 1491 if (auth_off <= 0) { 1492 if (auth_len > 0) { 1493 CVM_LOAD_MD5_UNIT(*pdata, next); 1494 CVM_LOAD_MD5_UNIT(*data, next); 1495 auth_len -= 16; 1496 } 1497 } else 1498 auth_off -= 16; 1499 1500 *pdata32[0] = mydata[0].data32[0]; 1501 *pdata32[1] = mydata[0].data32[1]; 1502 *pdata32[2] = mydata[1].data32[0]; 1503 *data32 = mydata[1].data32[1]; 1504 1505 IOV_CONSUME(iov, data32, data_i, data_l); 1506 } 1507 1508 /* finish the hash */ 1509 CVMX_PREFETCH0(od->octo_hmouter); 1510#if 0 1511 if (__predict_false(inplen)) { 1512 uint64_t tmp = 0; 1513 uint8_t *p = (uint8_t *) & tmp; 1514 p[inplen] = 0x80; 1515 do { 1516 inplen--; 1517 p[inplen] = ((uint8_t *) data)[inplen]; 1518 } while (inplen); 1519 CVM_LOAD_MD5_UNIT(tmp, next); 1520 } else { 1521 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1522 } 1523#else 1524 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1525#endif 1526 1527 /* Finish Inner hash */ 1528 while (next != 7) { 1529 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 1530 } 1531 CVMX_ES64(tmp1, ((alen + 64) << 3)); 1532 CVM_LOAD_MD5_UNIT(tmp1, next); 1533 1534 /* Get the inner hash of HMAC */ 1535 CVMX_MF_HSH_IV(tmp1, 0); 1536 CVMX_MF_HSH_IV(tmp2, 1); 1537 1538 /* Initialize hash unit */ 1539 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1540 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1541 1542 CVMX_MT_HSH_DAT(tmp1, 0); 1543 CVMX_MT_HSH_DAT(tmp2, 1); 1544 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 1545 CVMX_MT_HSH_DATZ(3); 1546 CVMX_MT_HSH_DATZ(4); 1547 CVMX_MT_HSH_DATZ(5); 1548 CVMX_MT_HSH_DATZ(6); 1549 CVMX_ES64(tmp1, ((64 + 16) << 3)); 1550 CVMX_MT_HSH_STARTMD5(tmp1); 1551 1552 /* save the HMAC */ 1553 IOV_INIT(iov, data32, data_i, data_l); 1554 while (icv_off > 0) { 1555 IOV_CONSUME(iov, data32, data_i, data_l); 1556 icv_off -= 4; 1557 } 1558 CVMX_MF_HSH_IV(tmp1, 0); 1559 *data32 = (uint32_t) (tmp1 >> 32); 1560 IOV_CONSUME(iov, data32, data_i, data_l); 1561 *data32 = (uint32_t) tmp1; 1562 IOV_CONSUME(iov, data32, data_i, data_l); 1563 CVMX_MF_HSH_IV(tmp1, 1); 1564 *data32 = (uint32_t) (tmp1 >> 32); 1565 1566 return 0; 1567} 1568 1569int 1570octo_aes_cbc_md5_decrypt( 1571 struct octo_sess *od, 1572 struct iovec *iov, size_t iovcnt, size_t iovlen, 1573 int auth_off, int auth_len, 1574 int crypt_off, int crypt_len, 1575 int icv_off, uint8_t *ivp) 1576{ 1577 register int next = 0; 1578 union { 1579 uint32_t data32[2]; 1580 uint64_t data64[1]; 1581 } mydata[2]; 1582 uint64_t *pdata = &mydata[0].data64[0]; 1583 uint64_t *data = &mydata[1].data64[0]; 1584 uint32_t *data32; 1585 uint64_t tmp1, tmp2; 1586 int data_i, data_l, alen = auth_len; 1587 1588 dprintf("%s()\n", __func__); 1589 1590 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1591 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1592 (crypt_len & 0x7) || 1593 (auth_len & 0x7) || 1594 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1595 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1596 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1597 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1598 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1599 return -EINVAL; 1600 } 1601 1602 IOV_INIT(iov, data32, data_i, data_l); 1603 1604 CVMX_PREFETCH0(ivp); 1605 CVMX_PREFETCH0(od->octo_enckey); 1606 1607 /* load AES Key */ 1608 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1609 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1610 1611 if (od->octo_encklen == 16) { 1612 CVMX_MT_AES_KEY(0x0, 2); 1613 CVMX_MT_AES_KEY(0x0, 3); 1614 } else if (od->octo_encklen == 24) { 1615 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1616 CVMX_MT_AES_KEY(0x0, 3); 1617 } else if (od->octo_encklen == 32) { 1618 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1619 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1620 } else { 1621 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1622 return -EINVAL; 1623 } 1624 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1625 1626 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1627 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 1628 1629 /* Load MD5 IV */ 1630 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1631 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1632 1633 while (crypt_off > 0 && auth_off > 0) { 1634 IOV_CONSUME(iov, data32, data_i, data_l); 1635 crypt_off -= 4; 1636 auth_off -= 4; 1637 } 1638 1639 while (crypt_len > 0 || auth_len > 0) { 1640 uint32_t *pdata32[3]; 1641 1642 pdata32[0] = data32; 1643 mydata[0].data32[0] = *data32; 1644 IOV_CONSUME(iov, data32, data_i, data_l); 1645 pdata32[1] = data32; 1646 mydata[0].data32[1] = *data32; 1647 IOV_CONSUME(iov, data32, data_i, data_l); 1648 pdata32[2] = data32; 1649 mydata[1].data32[0] = *data32; 1650 IOV_CONSUME(iov, data32, data_i, data_l); 1651 mydata[1].data32[1] = *data32; 1652 1653 if (auth_off <= 0) { 1654 if (auth_len > 0) { 1655 CVM_LOAD_MD5_UNIT(*pdata, next); 1656 CVM_LOAD_MD5_UNIT(*data, next); 1657 auth_len -= 16; 1658 } 1659 } else 1660 auth_off -= 16; 1661 1662 if (crypt_off <= 0) { 1663 if (crypt_len > 0) { 1664 CVMX_MT_AES_DEC_CBC0(*pdata); 1665 CVMX_MT_AES_DEC_CBC1(*data); 1666 CVMX_MF_AES_RESULT(*pdata, 0); 1667 CVMX_MF_AES_RESULT(*data, 1); 1668 crypt_len -= 16; 1669 } 1670 } else 1671 crypt_off -= 16; 1672 1673 *pdata32[0] = mydata[0].data32[0]; 1674 *pdata32[1] = mydata[0].data32[1]; 1675 *pdata32[2] = mydata[1].data32[0]; 1676 *data32 = mydata[1].data32[1]; 1677 1678 IOV_CONSUME(iov, data32, data_i, data_l); 1679 } 1680 1681 /* finish the hash */ 1682 CVMX_PREFETCH0(od->octo_hmouter); 1683#if 0 1684 if (__predict_false(inplen)) { 1685 uint64_t tmp = 0; 1686 uint8_t *p = (uint8_t *) & tmp; 1687 p[inplen] = 0x80; 1688 do { 1689 inplen--; 1690 p[inplen] = ((uint8_t *) data)[inplen]; 1691 } while (inplen); 1692 CVM_LOAD_MD5_UNIT(tmp, next); 1693 } else { 1694 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1695 } 1696#else 1697 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1698#endif 1699 1700 /* Finish Inner hash */ 1701 while (next != 7) { 1702 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); 1703 } 1704 CVMX_ES64(tmp1, ((alen + 64) << 3)); 1705 CVM_LOAD_MD5_UNIT(tmp1, next); 1706 1707 /* Get the inner hash of HMAC */ 1708 CVMX_MF_HSH_IV(tmp1, 0); 1709 CVMX_MF_HSH_IV(tmp2, 1); 1710 1711 /* Initialize hash unit */ 1712 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1713 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1714 1715 CVMX_MT_HSH_DAT(tmp1, 0); 1716 CVMX_MT_HSH_DAT(tmp2, 1); 1717 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); 1718 CVMX_MT_HSH_DATZ(3); 1719 CVMX_MT_HSH_DATZ(4); 1720 CVMX_MT_HSH_DATZ(5); 1721 CVMX_MT_HSH_DATZ(6); 1722 CVMX_ES64(tmp1, ((64 + 16) << 3)); 1723 CVMX_MT_HSH_STARTMD5(tmp1); 1724 1725 /* save the HMAC */ 1726 IOV_INIT(iov, data32, data_i, data_l); 1727 while (icv_off > 0) { 1728 IOV_CONSUME(iov, data32, data_i, data_l); 1729 icv_off -= 4; 1730 } 1731 CVMX_MF_HSH_IV(tmp1, 0); 1732 *data32 = (uint32_t) (tmp1 >> 32); 1733 IOV_CONSUME(iov, data32, data_i, data_l); 1734 *data32 = (uint32_t) tmp1; 1735 IOV_CONSUME(iov, data32, data_i, data_l); 1736 CVMX_MF_HSH_IV(tmp1, 1); 1737 *data32 = (uint32_t) (tmp1 >> 32); 1738 1739 return 0; 1740} 1741 1742/****************************************************************************/ 1743/* AES SHA1 */ 1744 1745int 1746octo_aes_cbc_sha1_encrypt( 1747 struct octo_sess *od, 1748 struct iovec *iov, size_t iovcnt, size_t iovlen, 1749 int auth_off, int auth_len, 1750 int crypt_off, int crypt_len, 1751 int icv_off, uint8_t *ivp) 1752{ 1753 register int next = 0; 1754 union { 1755 uint32_t data32[2]; 1756 uint64_t data64[1]; 1757 } mydata[2]; 1758 uint64_t *pdata = &mydata[0].data64[0]; 1759 uint64_t *data = &mydata[1].data64[0]; 1760 uint32_t *data32; 1761 uint64_t tmp1, tmp2, tmp3; 1762 int data_i, data_l, alen = auth_len; 1763 1764 dprintf("%s()\n", __func__); 1765 1766 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1767 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1768 (crypt_len & 0x7) || 1769 (auth_len & 0x7) || 1770 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1771 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1772 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1773 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1774 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1775 return -EINVAL; 1776 } 1777 1778 IOV_INIT(iov, data32, data_i, data_l); 1779 1780 CVMX_PREFETCH0(ivp); 1781 CVMX_PREFETCH0(od->octo_enckey); 1782 1783 /* load AES Key */ 1784 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1785 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1786 1787 if (od->octo_encklen == 16) { 1788 CVMX_MT_AES_KEY(0x0, 2); 1789 CVMX_MT_AES_KEY(0x0, 3); 1790 } else if (od->octo_encklen == 24) { 1791 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1792 CVMX_MT_AES_KEY(0x0, 3); 1793 } else if (od->octo_encklen == 32) { 1794 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1795 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1796 } else { 1797 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1798 return -EINVAL; 1799 } 1800 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1801 1802 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1803 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 1804 1805 /* Load SHA IV */ 1806 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 1807 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 1808 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 1809 1810 while (crypt_off > 0 && auth_off > 0) { 1811 IOV_CONSUME(iov, data32, data_i, data_l); 1812 crypt_off -= 4; 1813 auth_off -= 4; 1814 } 1815 1816 while (crypt_len > 0 || auth_len > 0) { 1817 uint32_t *pdata32[3]; 1818 1819 pdata32[0] = data32; 1820 mydata[0].data32[0] = *data32; 1821 IOV_CONSUME(iov, data32, data_i, data_l); 1822 pdata32[1] = data32; 1823 mydata[0].data32[1] = *data32; 1824 IOV_CONSUME(iov, data32, data_i, data_l); 1825 pdata32[2] = data32; 1826 mydata[1].data32[0] = *data32; 1827 IOV_CONSUME(iov, data32, data_i, data_l); 1828 mydata[1].data32[1] = *data32; 1829 1830 1831 if (crypt_off <= 0) { 1832 if (crypt_len > 0) { 1833 CVMX_MT_AES_ENC_CBC0(*pdata); 1834 CVMX_MT_AES_ENC_CBC1(*data); 1835 CVMX_MF_AES_RESULT(*pdata, 0); 1836 CVMX_MF_AES_RESULT(*data, 1); 1837 crypt_len -= 16; 1838 } 1839 } else 1840 crypt_off -= 16; 1841 1842 if (auth_off <= 0) { 1843 if (auth_len > 0) { 1844 CVM_LOAD_SHA_UNIT(*pdata, next); 1845 CVM_LOAD_SHA_UNIT(*data, next); 1846 auth_len -= 16; 1847 } 1848 } else 1849 auth_off -= 16; 1850 1851 *pdata32[0] = mydata[0].data32[0]; 1852 *pdata32[1] = mydata[0].data32[1]; 1853 *pdata32[2] = mydata[1].data32[0]; 1854 *data32 = mydata[1].data32[1]; 1855 1856 IOV_CONSUME(iov, data32, data_i, data_l); 1857 } 1858 1859 /* finish the hash */ 1860 CVMX_PREFETCH0(od->octo_hmouter); 1861#if 0 1862 if (__predict_false(inplen)) { 1863 uint64_t tmp = 0; 1864 uint8_t *p = (uint8_t *) & tmp; 1865 p[inplen] = 0x80; 1866 do { 1867 inplen--; 1868 p[inplen] = ((uint8_t *) data)[inplen]; 1869 } while (inplen); 1870 CVM_LOAD_SHA_UNIT(tmp, next); 1871 } else { 1872 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1873 } 1874#else 1875 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 1876#endif 1877 1878 /* Finish Inner hash */ 1879 while (next != 7) { 1880 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 1881 } 1882 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 1883 1884 /* Get the inner hash of HMAC */ 1885 CVMX_MF_HSH_IV(tmp1, 0); 1886 CVMX_MF_HSH_IV(tmp2, 1); 1887 tmp3 = 0; 1888 CVMX_MF_HSH_IV(tmp3, 2); 1889 1890 /* Initialize hash unit */ 1891 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 1892 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 1893 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 1894 1895 CVMX_MT_HSH_DAT(tmp1, 0); 1896 CVMX_MT_HSH_DAT(tmp2, 1); 1897 tmp3 |= 0x0000000080000000; 1898 CVMX_MT_HSH_DAT(tmp3, 2); 1899 CVMX_MT_HSH_DATZ(3); 1900 CVMX_MT_HSH_DATZ(4); 1901 CVMX_MT_HSH_DATZ(5); 1902 CVMX_MT_HSH_DATZ(6); 1903 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 1904 1905 /* finish the hash */ 1906 CVMX_PREFETCH0(od->octo_hmouter); 1907#if 0 1908 if (__predict_false(inplen)) { 1909 uint64_t tmp = 0; 1910 uint8_t *p = (uint8_t *) & tmp; 1911 p[inplen] = 0x80; 1912 do { 1913 inplen--; 1914 p[inplen] = ((uint8_t *) data)[inplen]; 1915 } while (inplen); 1916 CVM_LOAD_MD5_UNIT(tmp, next); 1917 } else { 1918 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1919 } 1920#else 1921 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 1922#endif 1923 1924 /* save the HMAC */ 1925 IOV_INIT(iov, data32, data_i, data_l); 1926 while (icv_off > 0) { 1927 IOV_CONSUME(iov, data32, data_i, data_l); 1928 icv_off -= 4; 1929 } 1930 CVMX_MF_HSH_IV(tmp1, 0); 1931 *data32 = (uint32_t) (tmp1 >> 32); 1932 IOV_CONSUME(iov, data32, data_i, data_l); 1933 *data32 = (uint32_t) tmp1; 1934 IOV_CONSUME(iov, data32, data_i, data_l); 1935 CVMX_MF_HSH_IV(tmp1, 1); 1936 *data32 = (uint32_t) (tmp1 >> 32); 1937 1938 return 0; 1939} 1940 1941int 1942octo_aes_cbc_sha1_decrypt( 1943 struct octo_sess *od, 1944 struct iovec *iov, size_t iovcnt, size_t iovlen, 1945 int auth_off, int auth_len, 1946 int crypt_off, int crypt_len, 1947 int icv_off, uint8_t *ivp) 1948{ 1949 register int next = 0; 1950 union { 1951 uint32_t data32[2]; 1952 uint64_t data64[1]; 1953 } mydata[2]; 1954 uint64_t *pdata = &mydata[0].data64[0]; 1955 uint64_t *data = &mydata[1].data64[0]; 1956 uint32_t *data32; 1957 uint64_t tmp1, tmp2, tmp3; 1958 int data_i, data_l, alen = auth_len; 1959 1960 dprintf("%s()\n", __func__); 1961 1962 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || 1963 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || 1964 (crypt_len & 0x7) || 1965 (auth_len & 0x7) || 1966 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { 1967 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " 1968 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " 1969 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, 1970 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); 1971 return -EINVAL; 1972 } 1973 1974 IOV_INIT(iov, data32, data_i, data_l); 1975 1976 CVMX_PREFETCH0(ivp); 1977 CVMX_PREFETCH0(od->octo_enckey); 1978 1979 /* load AES Key */ 1980 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); 1981 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); 1982 1983 if (od->octo_encklen == 16) { 1984 CVMX_MT_AES_KEY(0x0, 2); 1985 CVMX_MT_AES_KEY(0x0, 3); 1986 } else if (od->octo_encklen == 24) { 1987 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1988 CVMX_MT_AES_KEY(0x0, 3); 1989 } else if (od->octo_encklen == 32) { 1990 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); 1991 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); 1992 } else { 1993 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); 1994 return -EINVAL; 1995 } 1996 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); 1997 1998 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); 1999 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); 2000 2001 /* Load MD5 IV */ 2002 CVMX_MT_HSH_IV(od->octo_hminner[0], 0); 2003 CVMX_MT_HSH_IV(od->octo_hminner[1], 1); 2004 CVMX_MT_HSH_IV(od->octo_hminner[2], 2); 2005 2006 while (crypt_off > 0 && auth_off > 0) { 2007 IOV_CONSUME(iov, data32, data_i, data_l); 2008 crypt_off -= 4; 2009 auth_off -= 4; 2010 } 2011 2012 while (crypt_len > 0 || auth_len > 0) { 2013 uint32_t *pdata32[3]; 2014 2015 pdata32[0] = data32; 2016 mydata[0].data32[0] = *data32; 2017 IOV_CONSUME(iov, data32, data_i, data_l); 2018 pdata32[1] = data32; 2019 mydata[0].data32[1] = *data32; 2020 IOV_CONSUME(iov, data32, data_i, data_l); 2021 pdata32[2] = data32; 2022 mydata[1].data32[0] = *data32; 2023 IOV_CONSUME(iov, data32, data_i, data_l); 2024 mydata[1].data32[1] = *data32; 2025 2026 if (auth_off <= 0) { 2027 if (auth_len > 0) { 2028 CVM_LOAD_SHA_UNIT(*pdata, next); 2029 CVM_LOAD_SHA_UNIT(*data, next); 2030 auth_len -= 16; 2031 } 2032 } else 2033 auth_off -= 16; 2034 2035 if (crypt_off <= 0) { 2036 if (crypt_len > 0) { 2037 CVMX_MT_AES_DEC_CBC0(*pdata); 2038 CVMX_MT_AES_DEC_CBC1(*data); 2039 CVMX_MF_AES_RESULT(*pdata, 0); 2040 CVMX_MF_AES_RESULT(*data, 1); 2041 crypt_len -= 16; 2042 } 2043 } else 2044 crypt_off -= 16; 2045 2046 *pdata32[0] = mydata[0].data32[0]; 2047 *pdata32[1] = mydata[0].data32[1]; 2048 *pdata32[2] = mydata[1].data32[0]; 2049 *data32 = mydata[1].data32[1]; 2050 2051 IOV_CONSUME(iov, data32, data_i, data_l); 2052 } 2053 2054 /* finish the hash */ 2055 CVMX_PREFETCH0(od->octo_hmouter); 2056#if 0 2057 if (__predict_false(inplen)) { 2058 uint64_t tmp = 0; 2059 uint8_t *p = (uint8_t *) & tmp; 2060 p[inplen] = 0x80; 2061 do { 2062 inplen--; 2063 p[inplen] = ((uint8_t *) data)[inplen]; 2064 } while (inplen); 2065 CVM_LOAD_SHA_UNIT(tmp, next); 2066 } else { 2067 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 2068 } 2069#else 2070 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); 2071#endif 2072 2073 /* Finish Inner hash */ 2074 while (next != 7) { 2075 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); 2076 } 2077 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); 2078 2079 /* Get the inner hash of HMAC */ 2080 CVMX_MF_HSH_IV(tmp1, 0); 2081 CVMX_MF_HSH_IV(tmp2, 1); 2082 tmp3 = 0; 2083 CVMX_MF_HSH_IV(tmp3, 2); 2084 2085 /* Initialize hash unit */ 2086 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); 2087 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); 2088 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); 2089 2090 CVMX_MT_HSH_DAT(tmp1, 0); 2091 CVMX_MT_HSH_DAT(tmp2, 1); 2092 tmp3 |= 0x0000000080000000; 2093 CVMX_MT_HSH_DAT(tmp3, 2); 2094 CVMX_MT_HSH_DATZ(3); 2095 CVMX_MT_HSH_DATZ(4); 2096 CVMX_MT_HSH_DATZ(5); 2097 CVMX_MT_HSH_DATZ(6); 2098 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); 2099 2100 /* finish the hash */ 2101 CVMX_PREFETCH0(od->octo_hmouter); 2102#if 0 2103 if (__predict_false(inplen)) { 2104 uint64_t tmp = 0; 2105 uint8_t *p = (uint8_t *) & tmp; 2106 p[inplen] = 0x80; 2107 do { 2108 inplen--; 2109 p[inplen] = ((uint8_t *) data)[inplen]; 2110 } while (inplen); 2111 CVM_LOAD_MD5_UNIT(tmp, next); 2112 } else { 2113 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 2114 } 2115#else 2116 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); 2117#endif 2118 2119 /* save the HMAC */ 2120 IOV_INIT(iov, data32, data_i, data_l); 2121 while (icv_off > 0) { 2122 IOV_CONSUME(iov, data32, data_i, data_l); 2123 icv_off -= 4; 2124 } 2125 CVMX_MF_HSH_IV(tmp1, 0); 2126 *data32 = (uint32_t) (tmp1 >> 32); 2127 IOV_CONSUME(iov, data32, data_i, data_l); 2128 *data32 = (uint32_t) tmp1; 2129 IOV_CONSUME(iov, data32, data_i, data_l); 2130 CVMX_MF_HSH_IV(tmp1, 1); 2131 *data32 = (uint32_t) (tmp1 >> 32); 2132 2133 return 0; 2134} 2135 2136/****************************************************************************/ 2137