1#! /usr/bin/env perl 2# Copyright 2005-2016 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the OpenSSL license (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9# 10# ==================================================================== 11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL 12# project. The module is, however, dual licensed under OpenSSL and 13# CRYPTOGAMS licenses depending on where you obtain it. For further 14# details see http://www.openssl.org/~appro/cryptogams/. 15# ==================================================================== 16# 17# Version 2.1. 18# 19# aes-*-cbc benchmarks are improved by >70% [compared to gcc 3.3.2 on 20# Opteron 240 CPU] plus all the bells-n-whistles from 32-bit version 21# [you'll notice a lot of resemblance], such as compressed S-boxes 22# in little-endian byte order, prefetch of these tables in CBC mode, 23# as well as avoiding L1 cache aliasing between stack frame and key 24# schedule and already mentioned tables, compressed Td4... 25# 26# Performance in number of cycles per processed byte for 128-bit key: 27# 28# ECB encrypt ECB decrypt CBC large chunk 29# AMD64 33 43 13.0 30# EM64T 38 56 18.6(*) 31# Core 2 30 42 14.5(*) 32# Atom 65 86 32.1(*) 33# 34# (*) with hyper-threading off 35 36$flavour = shift; 37$output = shift; 38if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 39 40$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 41 42$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 43( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 44( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or 45die "can't locate x86_64-xlate.pl"; 46 47open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; 48*STDOUT=*OUT; 49 50$verticalspin=1; # unlike 32-bit version $verticalspin performs 51 # ~15% better on both AMD and Intel cores 52$speed_limit=512; # see aes-586.pl for details 53 54$code=".text\n"; 55 56$s0="%eax"; 57$s1="%ebx"; 58$s2="%ecx"; 59$s3="%edx"; 60$acc0="%esi"; $mask80="%rsi"; 61$acc1="%edi"; $maskfe="%rdi"; 62$acc2="%ebp"; $mask1b="%rbp"; 63$inp="%r8"; 64$out="%r9"; 65$t0="%r10d"; 66$t1="%r11d"; 67$t2="%r12d"; 68$rnds="%r13d"; 69$sbox="%r14"; 70$key="%r15"; 71 72sub hi() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1h/; $r; } 73sub lo() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1l/; 74 $r =~ s/%[er]([sd]i)/%\1l/; 75 $r =~ s/%(r[0-9]+)[d]?/%\1b/; $r; } 76sub LO() { my $r=shift; $r =~ s/%r([a-z]+)/%e\1/; 77 $r =~ s/%r([0-9]+)/%r\1d/; $r; } 78sub _data_word() 79{ my $i; 80 while(defined($i=shift)) { $code.=sprintf".long\t0x%08x,0x%08x\n",$i,$i; } 81} 82sub data_word() 83{ my $i; 84 my $last=pop(@_); 85 $code.=".long\t"; 86 while(defined($i=shift)) { $code.=sprintf"0x%08x,",$i; } 87 $code.=sprintf"0x%08x\n",$last; 88} 89 90sub data_byte() 91{ my $i; 92 my $last=pop(@_); 93 $code.=".byte\t"; 94 while(defined($i=shift)) { $code.=sprintf"0x%02x,",$i&0xff; } 95 $code.=sprintf"0x%02x\n",$last&0xff; 96} 97 98sub encvert() 99{ my $t3="%r8d"; # zaps $inp! 100 101$code.=<<___; 102 # favor 3-way issue Opteron pipeline... 103 movzb `&lo("$s0")`,$acc0 104 movzb `&lo("$s1")`,$acc1 105 movzb `&lo("$s2")`,$acc2 106 mov 0($sbox,$acc0,8),$t0 107 mov 0($sbox,$acc1,8),$t1 108 mov 0($sbox,$acc2,8),$t2 109 110 movzb `&hi("$s1")`,$acc0 111 movzb `&hi("$s2")`,$acc1 112 movzb `&lo("$s3")`,$acc2 113 xor 3($sbox,$acc0,8),$t0 114 xor 3($sbox,$acc1,8),$t1 115 mov 0($sbox,$acc2,8),$t3 116 117 movzb `&hi("$s3")`,$acc0 118 shr \$16,$s2 119 movzb `&hi("$s0")`,$acc2 120 xor 3($sbox,$acc0,8),$t2 121 shr \$16,$s3 122 xor 3($sbox,$acc2,8),$t3 123 124 shr \$16,$s1 125 lea 16($key),$key 126 shr \$16,$s0 127 128 movzb `&lo("$s2")`,$acc0 129 movzb `&lo("$s3")`,$acc1 130 movzb `&lo("$s0")`,$acc2 131 xor 2($sbox,$acc0,8),$t0 132 xor 2($sbox,$acc1,8),$t1 133 xor 2($sbox,$acc2,8),$t2 134 135 movzb `&hi("$s3")`,$acc0 136 movzb `&hi("$s0")`,$acc1 137 movzb `&lo("$s1")`,$acc2 138 xor 1($sbox,$acc0,8),$t0 139 xor 1($sbox,$acc1,8),$t1 140 xor 2($sbox,$acc2,8),$t3 141 142 mov 12($key),$s3 143 movzb `&hi("$s1")`,$acc1 144 movzb `&hi("$s2")`,$acc2 145 mov 0($key),$s0 146 xor 1($sbox,$acc1,8),$t2 147 xor 1($sbox,$acc2,8),$t3 148 149 mov 4($key),$s1 150 mov 8($key),$s2 151 xor $t0,$s0 152 xor $t1,$s1 153 xor $t2,$s2 154 xor $t3,$s3 155___ 156} 157 158sub enclastvert() 159{ my $t3="%r8d"; # zaps $inp! 160 161$code.=<<___; 162 movzb `&lo("$s0")`,$acc0 163 movzb `&lo("$s1")`,$acc1 164 movzb `&lo("$s2")`,$acc2 165 movzb 2($sbox,$acc0,8),$t0 166 movzb 2($sbox,$acc1,8),$t1 167 movzb 2($sbox,$acc2,8),$t2 168 169 movzb `&lo("$s3")`,$acc0 170 movzb `&hi("$s1")`,$acc1 171 movzb `&hi("$s2")`,$acc2 172 movzb 2($sbox,$acc0,8),$t3 173 mov 0($sbox,$acc1,8),$acc1 #$t0 174 mov 0($sbox,$acc2,8),$acc2 #$t1 175 176 and \$0x0000ff00,$acc1 177 and \$0x0000ff00,$acc2 178 179 xor $acc1,$t0 180 xor $acc2,$t1 181 shr \$16,$s2 182 183 movzb `&hi("$s3")`,$acc0 184 movzb `&hi("$s0")`,$acc1 185 shr \$16,$s3 186 mov 0($sbox,$acc0,8),$acc0 #$t2 187 mov 0($sbox,$acc1,8),$acc1 #$t3 188 189 and \$0x0000ff00,$acc0 190 and \$0x0000ff00,$acc1 191 shr \$16,$s1 192 xor $acc0,$t2 193 xor $acc1,$t3 194 shr \$16,$s0 195 196 movzb `&lo("$s2")`,$acc0 197 movzb `&lo("$s3")`,$acc1 198 movzb `&lo("$s0")`,$acc2 199 mov 0($sbox,$acc0,8),$acc0 #$t0 200 mov 0($sbox,$acc1,8),$acc1 #$t1 201 mov 0($sbox,$acc2,8),$acc2 #$t2 202 203 and \$0x00ff0000,$acc0 204 and \$0x00ff0000,$acc1 205 and \$0x00ff0000,$acc2 206 207 xor $acc0,$t0 208 xor $acc1,$t1 209 xor $acc2,$t2 210 211 movzb `&lo("$s1")`,$acc0 212 movzb `&hi("$s3")`,$acc1 213 movzb `&hi("$s0")`,$acc2 214 mov 0($sbox,$acc0,8),$acc0 #$t3 215 mov 2($sbox,$acc1,8),$acc1 #$t0 216 mov 2($sbox,$acc2,8),$acc2 #$t1 217 218 and \$0x00ff0000,$acc0 219 and \$0xff000000,$acc1 220 and \$0xff000000,$acc2 221 222 xor $acc0,$t3 223 xor $acc1,$t0 224 xor $acc2,$t1 225 226 movzb `&hi("$s1")`,$acc0 227 movzb `&hi("$s2")`,$acc1 228 mov 16+12($key),$s3 229 mov 2($sbox,$acc0,8),$acc0 #$t2 230 mov 2($sbox,$acc1,8),$acc1 #$t3 231 mov 16+0($key),$s0 232 233 and \$0xff000000,$acc0 234 and \$0xff000000,$acc1 235 236 xor $acc0,$t2 237 xor $acc1,$t3 238 239 mov 16+4($key),$s1 240 mov 16+8($key),$s2 241 xor $t0,$s0 242 xor $t1,$s1 243 xor $t2,$s2 244 xor $t3,$s3 245___ 246} 247 248sub encstep() 249{ my ($i,@s) = @_; 250 my $tmp0=$acc0; 251 my $tmp1=$acc1; 252 my $tmp2=$acc2; 253 my $out=($t0,$t1,$t2,$s[0])[$i]; 254 255 if ($i==3) { 256 $tmp0=$s[1]; 257 $tmp1=$s[2]; 258 $tmp2=$s[3]; 259 } 260 $code.=" movzb ".&lo($s[0]).",$out\n"; 261 $code.=" mov $s[2],$tmp1\n" if ($i!=3); 262 $code.=" lea 16($key),$key\n" if ($i==0); 263 264 $code.=" movzb ".&hi($s[1]).",$tmp0\n"; 265 $code.=" mov 0($sbox,$out,8),$out\n"; 266 267 $code.=" shr \$16,$tmp1\n"; 268 $code.=" mov $s[3],$tmp2\n" if ($i!=3); 269 $code.=" xor 3($sbox,$tmp0,8),$out\n"; 270 271 $code.=" movzb ".&lo($tmp1).",$tmp1\n"; 272 $code.=" shr \$24,$tmp2\n"; 273 $code.=" xor 4*$i($key),$out\n"; 274 275 $code.=" xor 2($sbox,$tmp1,8),$out\n"; 276 $code.=" xor 1($sbox,$tmp2,8),$out\n"; 277 278 $code.=" mov $t0,$s[1]\n" if ($i==3); 279 $code.=" mov $t1,$s[2]\n" if ($i==3); 280 $code.=" mov $t2,$s[3]\n" if ($i==3); 281 $code.="\n"; 282} 283 284sub enclast() 285{ my ($i,@s)=@_; 286 my $tmp0=$acc0; 287 my $tmp1=$acc1; 288 my $tmp2=$acc2; 289 my $out=($t0,$t1,$t2,$s[0])[$i]; 290 291 if ($i==3) { 292 $tmp0=$s[1]; 293 $tmp1=$s[2]; 294 $tmp2=$s[3]; 295 } 296 $code.=" movzb ".&lo($s[0]).",$out\n"; 297 $code.=" mov $s[2],$tmp1\n" if ($i!=3); 298 299 $code.=" mov 2($sbox,$out,8),$out\n"; 300 $code.=" shr \$16,$tmp1\n"; 301 $code.=" mov $s[3],$tmp2\n" if ($i!=3); 302 303 $code.=" and \$0x000000ff,$out\n"; 304 $code.=" movzb ".&hi($s[1]).",$tmp0\n"; 305 $code.=" movzb ".&lo($tmp1).",$tmp1\n"; 306 $code.=" shr \$24,$tmp2\n"; 307 308 $code.=" mov 0($sbox,$tmp0,8),$tmp0\n"; 309 $code.=" mov 0($sbox,$tmp1,8),$tmp1\n"; 310 $code.=" mov 2($sbox,$tmp2,8),$tmp2\n"; 311 312 $code.=" and \$0x0000ff00,$tmp0\n"; 313 $code.=" and \$0x00ff0000,$tmp1\n"; 314 $code.=" and \$0xff000000,$tmp2\n"; 315 316 $code.=" xor $tmp0,$out\n"; 317 $code.=" mov $t0,$s[1]\n" if ($i==3); 318 $code.=" xor $tmp1,$out\n"; 319 $code.=" mov $t1,$s[2]\n" if ($i==3); 320 $code.=" xor $tmp2,$out\n"; 321 $code.=" mov $t2,$s[3]\n" if ($i==3); 322 $code.="\n"; 323} 324 325$code.=<<___; 326.type _x86_64_AES_encrypt,\@abi-omnipotent 327.align 16 328_x86_64_AES_encrypt: 329 xor 0($key),$s0 # xor with key 330 xor 4($key),$s1 331 xor 8($key),$s2 332 xor 12($key),$s3 333 334 mov 240($key),$rnds # load key->rounds 335 sub \$1,$rnds 336 jmp .Lenc_loop 337.align 16 338.Lenc_loop: 339___ 340 if ($verticalspin) { &encvert(); } 341 else { &encstep(0,$s0,$s1,$s2,$s3); 342 &encstep(1,$s1,$s2,$s3,$s0); 343 &encstep(2,$s2,$s3,$s0,$s1); 344 &encstep(3,$s3,$s0,$s1,$s2); 345 } 346$code.=<<___; 347 sub \$1,$rnds 348 jnz .Lenc_loop 349___ 350 if ($verticalspin) { &enclastvert(); } 351 else { &enclast(0,$s0,$s1,$s2,$s3); 352 &enclast(1,$s1,$s2,$s3,$s0); 353 &enclast(2,$s2,$s3,$s0,$s1); 354 &enclast(3,$s3,$s0,$s1,$s2); 355 $code.=<<___; 356 xor 16+0($key),$s0 # xor with key 357 xor 16+4($key),$s1 358 xor 16+8($key),$s2 359 xor 16+12($key),$s3 360___ 361 } 362$code.=<<___; 363 .byte 0xf3,0xc3 # rep ret 364.size _x86_64_AES_encrypt,.-_x86_64_AES_encrypt 365___ 366 367# it's possible to implement this by shifting tN by 8, filling least 368# significant byte with byte load and finally bswap-ing at the end, 369# but such partial register load kills Core 2... 370sub enccompactvert() 371{ my ($t3,$t4,$t5)=("%r8d","%r9d","%r13d"); 372 373$code.=<<___; 374 movzb `&lo("$s0")`,$t0 375 movzb `&lo("$s1")`,$t1 376 movzb `&lo("$s2")`,$t2 377 movzb `&lo("$s3")`,$t3 378 movzb `&hi("$s1")`,$acc0 379 movzb `&hi("$s2")`,$acc1 380 shr \$16,$s2 381 movzb `&hi("$s3")`,$acc2 382 movzb ($sbox,$t0,1),$t0 383 movzb ($sbox,$t1,1),$t1 384 movzb ($sbox,$t2,1),$t2 385 movzb ($sbox,$t3,1),$t3 386 387 movzb ($sbox,$acc0,1),$t4 #$t0 388 movzb `&hi("$s0")`,$acc0 389 movzb ($sbox,$acc1,1),$t5 #$t1 390 movzb `&lo("$s2")`,$acc1 391 movzb ($sbox,$acc2,1),$acc2 #$t2 392 movzb ($sbox,$acc0,1),$acc0 #$t3 393 394 shl \$8,$t4 395 shr \$16,$s3 396 shl \$8,$t5 397 xor $t4,$t0 398 shr \$16,$s0 399 movzb `&lo("$s3")`,$t4 400 shr \$16,$s1 401 xor $t5,$t1 402 shl \$8,$acc2 403 movzb `&lo("$s0")`,$t5 404 movzb ($sbox,$acc1,1),$acc1 #$t0 405 xor $acc2,$t2 406 407 shl \$8,$acc0 408 movzb `&lo("$s1")`,$acc2 409 shl \$16,$acc1 410 xor $acc0,$t3 411 movzb ($sbox,$t4,1),$t4 #$t1 412 movzb `&hi("$s3")`,$acc0 413 movzb ($sbox,$t5,1),$t5 #$t2 414 xor $acc1,$t0 415 416 shr \$8,$s2 417 movzb `&hi("$s0")`,$acc1 418 shl \$16,$t4 419 shr \$8,$s1 420 shl \$16,$t5 421 xor $t4,$t1 422 movzb ($sbox,$acc2,1),$acc2 #$t3 423 movzb ($sbox,$acc0,1),$acc0 #$t0 424 movzb ($sbox,$acc1,1),$acc1 #$t1 425 movzb ($sbox,$s2,1),$s3 #$t3 426 movzb ($sbox,$s1,1),$s2 #$t2 427 428 shl \$16,$acc2 429 xor $t5,$t2 430 shl \$24,$acc0 431 xor $acc2,$t3 432 shl \$24,$acc1 433 xor $acc0,$t0 434 shl \$24,$s3 435 xor $acc1,$t1 436 shl \$24,$s2 437 mov $t0,$s0 438 mov $t1,$s1 439 xor $t2,$s2 440 xor $t3,$s3 441___ 442} 443 444sub enctransform_ref() 445{ my $sn = shift; 446 my ($acc,$r2,$tmp)=("%r8d","%r9d","%r13d"); 447 448$code.=<<___; 449 mov $sn,$acc 450 and \$0x80808080,$acc 451 mov $acc,$tmp 452 shr \$7,$tmp 453 lea ($sn,$sn),$r2 454 sub $tmp,$acc 455 and \$0xfefefefe,$r2 456 and \$0x1b1b1b1b,$acc 457 mov $sn,$tmp 458 xor $acc,$r2 459 460 xor $r2,$sn 461 rol \$24,$sn 462 xor $r2,$sn 463 ror \$16,$tmp 464 xor $tmp,$sn 465 ror \$8,$tmp 466 xor $tmp,$sn 467___ 468} 469 470# unlike decrypt case it does not pay off to parallelize enctransform 471sub enctransform() 472{ my ($t3,$r20,$r21)=($acc2,"%r8d","%r9d"); 473 474$code.=<<___; 475 mov \$0x80808080,$t0 476 mov \$0x80808080,$t1 477 and $s0,$t0 478 and $s1,$t1 479 mov $t0,$acc0 480 mov $t1,$acc1 481 shr \$7,$t0 482 lea ($s0,$s0),$r20 483 shr \$7,$t1 484 lea ($s1,$s1),$r21 485 sub $t0,$acc0 486 sub $t1,$acc1 487 and \$0xfefefefe,$r20 488 and \$0xfefefefe,$r21 489 and \$0x1b1b1b1b,$acc0 490 and \$0x1b1b1b1b,$acc1 491 mov $s0,$t0 492 mov $s1,$t1 493 xor $acc0,$r20 494 xor $acc1,$r21 495 496 xor $r20,$s0 497 xor $r21,$s1 498 mov \$0x80808080,$t2 499 rol \$24,$s0 500 mov \$0x80808080,$t3 501 rol \$24,$s1 502 and $s2,$t2 503 and $s3,$t3 504 xor $r20,$s0 505 xor $r21,$s1 506 mov $t2,$acc0 507 ror \$16,$t0 508 mov $t3,$acc1 509 ror \$16,$t1 510 lea ($s2,$s2),$r20 511 shr \$7,$t2 512 xor $t0,$s0 513 shr \$7,$t3 514 xor $t1,$s1 515 ror \$8,$t0 516 lea ($s3,$s3),$r21 517 ror \$8,$t1 518 sub $t2,$acc0 519 sub $t3,$acc1 520 xor $t0,$s0 521 xor $t1,$s1 522 523 and \$0xfefefefe,$r20 524 and \$0xfefefefe,$r21 525 and \$0x1b1b1b1b,$acc0 526 and \$0x1b1b1b1b,$acc1 527 mov $s2,$t2 528 mov $s3,$t3 529 xor $acc0,$r20 530 xor $acc1,$r21 531 532 ror \$16,$t2 533 xor $r20,$s2 534 ror \$16,$t3 535 xor $r21,$s3 536 rol \$24,$s2 537 mov 0($sbox),$acc0 # prefetch Te4 538 rol \$24,$s3 539 xor $r20,$s2 540 mov 64($sbox),$acc1 541 xor $r21,$s3 542 mov 128($sbox),$r20 543 xor $t2,$s2 544 ror \$8,$t2 545 xor $t3,$s3 546 ror \$8,$t3 547 xor $t2,$s2 548 mov 192($sbox),$r21 549 xor $t3,$s3 550___ 551} 552 553$code.=<<___; 554.type _x86_64_AES_encrypt_compact,\@abi-omnipotent 555.align 16 556_x86_64_AES_encrypt_compact: 557 lea 128($sbox),$inp # size optimization 558 mov 0-128($inp),$acc1 # prefetch Te4 559 mov 32-128($inp),$acc2 560 mov 64-128($inp),$t0 561 mov 96-128($inp),$t1 562 mov 128-128($inp),$acc1 563 mov 160-128($inp),$acc2 564 mov 192-128($inp),$t0 565 mov 224-128($inp),$t1 566 jmp .Lenc_loop_compact 567.align 16 568.Lenc_loop_compact: 569 xor 0($key),$s0 # xor with key 570 xor 4($key),$s1 571 xor 8($key),$s2 572 xor 12($key),$s3 573 lea 16($key),$key 574___ 575 &enccompactvert(); 576$code.=<<___; 577 cmp 16(%rsp),$key 578 je .Lenc_compact_done 579___ 580 &enctransform(); 581$code.=<<___; 582 jmp .Lenc_loop_compact 583.align 16 584.Lenc_compact_done: 585 xor 0($key),$s0 586 xor 4($key),$s1 587 xor 8($key),$s2 588 xor 12($key),$s3 589 .byte 0xf3,0xc3 # rep ret 590.size _x86_64_AES_encrypt_compact,.-_x86_64_AES_encrypt_compact 591___ 592 593# void aes_nohw_encrypt (const void *inp,void *out,const AES_KEY *key); 594$code.=<<___; 595.align 16 596.globl aes_nohw_encrypt 597.type aes_nohw_encrypt,\@function,3 598.hidden aes_nohw_encrypt 599aes_nohw_encrypt: 600.cfi_startproc 601 mov %rsp,%rax 602.cfi_def_cfa_register %rax 603 push %rbx 604.cfi_push %rbx 605 push %rbp 606.cfi_push %rbp 607 push %r12 608.cfi_push %r12 609 push %r13 610.cfi_push %r13 611 push %r14 612.cfi_push %r14 613 push %r15 614.cfi_push %r15 615 616 # allocate frame "above" key schedule 617 lea -63(%rdx),%rcx # %rdx is key argument 618 and \$-64,%rsp 619 sub %rsp,%rcx 620 neg %rcx 621 and \$0x3c0,%rcx 622 sub %rcx,%rsp 623 sub \$32,%rsp 624 625 mov %rsi,16(%rsp) # save out 626 mov %rax,24(%rsp) # save original stack pointer 627.cfi_cfa_expression %rsp+24,deref,+8 628.Lenc_prologue: 629 630 mov %rdx,$key 631 mov 240($key),$rnds # load rounds 632 633 mov 0(%rdi),$s0 # load input vector 634 mov 4(%rdi),$s1 635 mov 8(%rdi),$s2 636 mov 12(%rdi),$s3 637 638 shl \$4,$rnds 639 lea ($key,$rnds),%rbp 640 mov $key,(%rsp) # key schedule 641 mov %rbp,8(%rsp) # end of key schedule 642 643 # pick Te4 copy which can't "overlap" with stack frame or key schedule 644 lea .LAES_Te+2048(%rip),$sbox 645 lea 768(%rsp),%rbp 646 sub $sbox,%rbp 647 and \$0x300,%rbp 648 lea ($sbox,%rbp),$sbox 649 650 call _x86_64_AES_encrypt_compact 651 652 mov 16(%rsp),$out # restore out 653 mov 24(%rsp),%rsi # restore saved stack pointer 654.cfi_def_cfa %rsi,8 655 mov $s0,0($out) # write output vector 656 mov $s1,4($out) 657 mov $s2,8($out) 658 mov $s3,12($out) 659 660 mov -48(%rsi),%r15 661.cfi_restore %r15 662 mov -40(%rsi),%r14 663.cfi_restore %r14 664 mov -32(%rsi),%r13 665.cfi_restore %r13 666 mov -24(%rsi),%r12 667.cfi_restore %r12 668 mov -16(%rsi),%rbp 669.cfi_restore %rbp 670 mov -8(%rsi),%rbx 671.cfi_restore %rbx 672 lea (%rsi),%rsp 673.cfi_def_cfa_register %rsp 674.Lenc_epilogue: 675 ret 676.cfi_endproc 677.size aes_nohw_encrypt,.-aes_nohw_encrypt 678___ 679 680#------------------------------------------------------------------# 681 682sub decvert() 683{ my $t3="%r8d"; # zaps $inp! 684 685$code.=<<___; 686 # favor 3-way issue Opteron pipeline... 687 movzb `&lo("$s0")`,$acc0 688 movzb `&lo("$s1")`,$acc1 689 movzb `&lo("$s2")`,$acc2 690 mov 0($sbox,$acc0,8),$t0 691 mov 0($sbox,$acc1,8),$t1 692 mov 0($sbox,$acc2,8),$t2 693 694 movzb `&hi("$s3")`,$acc0 695 movzb `&hi("$s0")`,$acc1 696 movzb `&lo("$s3")`,$acc2 697 xor 3($sbox,$acc0,8),$t0 698 xor 3($sbox,$acc1,8),$t1 699 mov 0($sbox,$acc2,8),$t3 700 701 movzb `&hi("$s1")`,$acc0 702 shr \$16,$s0 703 movzb `&hi("$s2")`,$acc2 704 xor 3($sbox,$acc0,8),$t2 705 shr \$16,$s3 706 xor 3($sbox,$acc2,8),$t3 707 708 shr \$16,$s1 709 lea 16($key),$key 710 shr \$16,$s2 711 712 movzb `&lo("$s2")`,$acc0 713 movzb `&lo("$s3")`,$acc1 714 movzb `&lo("$s0")`,$acc2 715 xor 2($sbox,$acc0,8),$t0 716 xor 2($sbox,$acc1,8),$t1 717 xor 2($sbox,$acc2,8),$t2 718 719 movzb `&hi("$s1")`,$acc0 720 movzb `&hi("$s2")`,$acc1 721 movzb `&lo("$s1")`,$acc2 722 xor 1($sbox,$acc0,8),$t0 723 xor 1($sbox,$acc1,8),$t1 724 xor 2($sbox,$acc2,8),$t3 725 726 movzb `&hi("$s3")`,$acc0 727 mov 12($key),$s3 728 movzb `&hi("$s0")`,$acc2 729 xor 1($sbox,$acc0,8),$t2 730 mov 0($key),$s0 731 xor 1($sbox,$acc2,8),$t3 732 733 xor $t0,$s0 734 mov 4($key),$s1 735 mov 8($key),$s2 736 xor $t2,$s2 737 xor $t1,$s1 738 xor $t3,$s3 739___ 740} 741 742sub declastvert() 743{ my $t3="%r8d"; # zaps $inp! 744 745$code.=<<___; 746 lea 2048($sbox),$sbox # size optimization 747 movzb `&lo("$s0")`,$acc0 748 movzb `&lo("$s1")`,$acc1 749 movzb `&lo("$s2")`,$acc2 750 movzb ($sbox,$acc0,1),$t0 751 movzb ($sbox,$acc1,1),$t1 752 movzb ($sbox,$acc2,1),$t2 753 754 movzb `&lo("$s3")`,$acc0 755 movzb `&hi("$s3")`,$acc1 756 movzb `&hi("$s0")`,$acc2 757 movzb ($sbox,$acc0,1),$t3 758 movzb ($sbox,$acc1,1),$acc1 #$t0 759 movzb ($sbox,$acc2,1),$acc2 #$t1 760 761 shl \$8,$acc1 762 shl \$8,$acc2 763 764 xor $acc1,$t0 765 xor $acc2,$t1 766 shr \$16,$s3 767 768 movzb `&hi("$s1")`,$acc0 769 movzb `&hi("$s2")`,$acc1 770 shr \$16,$s0 771 movzb ($sbox,$acc0,1),$acc0 #$t2 772 movzb ($sbox,$acc1,1),$acc1 #$t3 773 774 shl \$8,$acc0 775 shl \$8,$acc1 776 shr \$16,$s1 777 xor $acc0,$t2 778 xor $acc1,$t3 779 shr \$16,$s2 780 781 movzb `&lo("$s2")`,$acc0 782 movzb `&lo("$s3")`,$acc1 783 movzb `&lo("$s0")`,$acc2 784 movzb ($sbox,$acc0,1),$acc0 #$t0 785 movzb ($sbox,$acc1,1),$acc1 #$t1 786 movzb ($sbox,$acc2,1),$acc2 #$t2 787 788 shl \$16,$acc0 789 shl \$16,$acc1 790 shl \$16,$acc2 791 792 xor $acc0,$t0 793 xor $acc1,$t1 794 xor $acc2,$t2 795 796 movzb `&lo("$s1")`,$acc0 797 movzb `&hi("$s1")`,$acc1 798 movzb `&hi("$s2")`,$acc2 799 movzb ($sbox,$acc0,1),$acc0 #$t3 800 movzb ($sbox,$acc1,1),$acc1 #$t0 801 movzb ($sbox,$acc2,1),$acc2 #$t1 802 803 shl \$16,$acc0 804 shl \$24,$acc1 805 shl \$24,$acc2 806 807 xor $acc0,$t3 808 xor $acc1,$t0 809 xor $acc2,$t1 810 811 movzb `&hi("$s3")`,$acc0 812 movzb `&hi("$s0")`,$acc1 813 mov 16+12($key),$s3 814 movzb ($sbox,$acc0,1),$acc0 #$t2 815 movzb ($sbox,$acc1,1),$acc1 #$t3 816 mov 16+0($key),$s0 817 818 shl \$24,$acc0 819 shl \$24,$acc1 820 821 xor $acc0,$t2 822 xor $acc1,$t3 823 824 mov 16+4($key),$s1 825 mov 16+8($key),$s2 826 lea -2048($sbox),$sbox 827 xor $t0,$s0 828 xor $t1,$s1 829 xor $t2,$s2 830 xor $t3,$s3 831___ 832} 833 834sub decstep() 835{ my ($i,@s) = @_; 836 my $tmp0=$acc0; 837 my $tmp1=$acc1; 838 my $tmp2=$acc2; 839 my $out=($t0,$t1,$t2,$s[0])[$i]; 840 841 $code.=" mov $s[0],$out\n" if ($i!=3); 842 $tmp1=$s[2] if ($i==3); 843 $code.=" mov $s[2],$tmp1\n" if ($i!=3); 844 $code.=" and \$0xFF,$out\n"; 845 846 $code.=" mov 0($sbox,$out,8),$out\n"; 847 $code.=" shr \$16,$tmp1\n"; 848 $tmp2=$s[3] if ($i==3); 849 $code.=" mov $s[3],$tmp2\n" if ($i!=3); 850 851 $tmp0=$s[1] if ($i==3); 852 $code.=" movzb ".&hi($s[1]).",$tmp0\n"; 853 $code.=" and \$0xFF,$tmp1\n"; 854 $code.=" shr \$24,$tmp2\n"; 855 856 $code.=" xor 3($sbox,$tmp0,8),$out\n"; 857 $code.=" xor 2($sbox,$tmp1,8),$out\n"; 858 $code.=" xor 1($sbox,$tmp2,8),$out\n"; 859 860 $code.=" mov $t2,$s[1]\n" if ($i==3); 861 $code.=" mov $t1,$s[2]\n" if ($i==3); 862 $code.=" mov $t0,$s[3]\n" if ($i==3); 863 $code.="\n"; 864} 865 866sub declast() 867{ my ($i,@s)=@_; 868 my $tmp0=$acc0; 869 my $tmp1=$acc1; 870 my $tmp2=$acc2; 871 my $out=($t0,$t1,$t2,$s[0])[$i]; 872 873 $code.=" mov $s[0],$out\n" if ($i!=3); 874 $tmp1=$s[2] if ($i==3); 875 $code.=" mov $s[2],$tmp1\n" if ($i!=3); 876 $code.=" and \$0xFF,$out\n"; 877 878 $code.=" movzb 2048($sbox,$out,1),$out\n"; 879 $code.=" shr \$16,$tmp1\n"; 880 $tmp2=$s[3] if ($i==3); 881 $code.=" mov $s[3],$tmp2\n" if ($i!=3); 882 883 $tmp0=$s[1] if ($i==3); 884 $code.=" movzb ".&hi($s[1]).",$tmp0\n"; 885 $code.=" and \$0xFF,$tmp1\n"; 886 $code.=" shr \$24,$tmp2\n"; 887 888 $code.=" movzb 2048($sbox,$tmp0,1),$tmp0\n"; 889 $code.=" movzb 2048($sbox,$tmp1,1),$tmp1\n"; 890 $code.=" movzb 2048($sbox,$tmp2,1),$tmp2\n"; 891 892 $code.=" shl \$8,$tmp0\n"; 893 $code.=" shl \$16,$tmp1\n"; 894 $code.=" shl \$24,$tmp2\n"; 895 896 $code.=" xor $tmp0,$out\n"; 897 $code.=" mov $t2,$s[1]\n" if ($i==3); 898 $code.=" xor $tmp1,$out\n"; 899 $code.=" mov $t1,$s[2]\n" if ($i==3); 900 $code.=" xor $tmp2,$out\n"; 901 $code.=" mov $t0,$s[3]\n" if ($i==3); 902 $code.="\n"; 903} 904 905$code.=<<___; 906.type _x86_64_AES_decrypt,\@abi-omnipotent 907.align 16 908_x86_64_AES_decrypt: 909 xor 0($key),$s0 # xor with key 910 xor 4($key),$s1 911 xor 8($key),$s2 912 xor 12($key),$s3 913 914 mov 240($key),$rnds # load key->rounds 915 sub \$1,$rnds 916 jmp .Ldec_loop 917.align 16 918.Ldec_loop: 919___ 920 if ($verticalspin) { &decvert(); } 921 else { &decstep(0,$s0,$s3,$s2,$s1); 922 &decstep(1,$s1,$s0,$s3,$s2); 923 &decstep(2,$s2,$s1,$s0,$s3); 924 &decstep(3,$s3,$s2,$s1,$s0); 925 $code.=<<___; 926 lea 16($key),$key 927 xor 0($key),$s0 # xor with key 928 xor 4($key),$s1 929 xor 8($key),$s2 930 xor 12($key),$s3 931___ 932 } 933$code.=<<___; 934 sub \$1,$rnds 935 jnz .Ldec_loop 936___ 937 if ($verticalspin) { &declastvert(); } 938 else { &declast(0,$s0,$s3,$s2,$s1); 939 &declast(1,$s1,$s0,$s3,$s2); 940 &declast(2,$s2,$s1,$s0,$s3); 941 &declast(3,$s3,$s2,$s1,$s0); 942 $code.=<<___; 943 xor 16+0($key),$s0 # xor with key 944 xor 16+4($key),$s1 945 xor 16+8($key),$s2 946 xor 16+12($key),$s3 947___ 948 } 949$code.=<<___; 950 .byte 0xf3,0xc3 # rep ret 951.size _x86_64_AES_decrypt,.-_x86_64_AES_decrypt 952___ 953 954sub deccompactvert() 955{ my ($t3,$t4,$t5)=("%r8d","%r9d","%r13d"); 956 957$code.=<<___; 958 movzb `&lo("$s0")`,$t0 959 movzb `&lo("$s1")`,$t1 960 movzb `&lo("$s2")`,$t2 961 movzb `&lo("$s3")`,$t3 962 movzb `&hi("$s3")`,$acc0 963 movzb `&hi("$s0")`,$acc1 964 shr \$16,$s3 965 movzb `&hi("$s1")`,$acc2 966 movzb ($sbox,$t0,1),$t0 967 movzb ($sbox,$t1,1),$t1 968 movzb ($sbox,$t2,1),$t2 969 movzb ($sbox,$t3,1),$t3 970 971 movzb ($sbox,$acc0,1),$t4 #$t0 972 movzb `&hi("$s2")`,$acc0 973 movzb ($sbox,$acc1,1),$t5 #$t1 974 movzb ($sbox,$acc2,1),$acc2 #$t2 975 movzb ($sbox,$acc0,1),$acc0 #$t3 976 977 shr \$16,$s2 978 shl \$8,$t5 979 shl \$8,$t4 980 movzb `&lo("$s2")`,$acc1 981 shr \$16,$s0 982 xor $t4,$t0 983 shr \$16,$s1 984 movzb `&lo("$s3")`,$t4 985 986 shl \$8,$acc2 987 xor $t5,$t1 988 shl \$8,$acc0 989 movzb `&lo("$s0")`,$t5 990 movzb ($sbox,$acc1,1),$acc1 #$t0 991 xor $acc2,$t2 992 movzb `&lo("$s1")`,$acc2 993 994 shl \$16,$acc1 995 xor $acc0,$t3 996 movzb ($sbox,$t4,1),$t4 #$t1 997 movzb `&hi("$s1")`,$acc0 998 movzb ($sbox,$acc2,1),$acc2 #$t3 999 xor $acc1,$t0 1000 movzb ($sbox,$t5,1),$t5 #$t2 1001 movzb `&hi("$s2")`,$acc1 1002 1003 shl \$16,$acc2 1004 shl \$16,$t4 1005 shl \$16,$t5 1006 xor $acc2,$t3 1007 movzb `&hi("$s3")`,$acc2 1008 xor $t4,$t1 1009 shr \$8,$s0 1010 xor $t5,$t2 1011 1012 movzb ($sbox,$acc0,1),$acc0 #$t0 1013 movzb ($sbox,$acc1,1),$s1 #$t1 1014 movzb ($sbox,$acc2,1),$s2 #$t2 1015 movzb ($sbox,$s0,1),$s3 #$t3 1016 1017 mov $t0,$s0 1018 shl \$24,$acc0 1019 shl \$24,$s1 1020 shl \$24,$s2 1021 xor $acc0,$s0 1022 shl \$24,$s3 1023 xor $t1,$s1 1024 xor $t2,$s2 1025 xor $t3,$s3 1026___ 1027} 1028 1029# parallelized version! input is pair of 64-bit values: %rax=s1.s0 1030# and %rcx=s3.s2, output is four 32-bit values in %eax=s0, %ebx=s1, 1031# %ecx=s2 and %edx=s3. 1032sub dectransform() 1033{ my ($tp10,$tp20,$tp40,$tp80,$acc0)=("%rax","%r8", "%r9", "%r10","%rbx"); 1034 my ($tp18,$tp28,$tp48,$tp88,$acc8)=("%rcx","%r11","%r12","%r13","%rdx"); 1035 my $prefetch = shift; 1036 1037$code.=<<___; 1038 mov $mask80,$tp40 1039 mov $mask80,$tp48 1040 and $tp10,$tp40 1041 and $tp18,$tp48 1042 mov $tp40,$acc0 1043 mov $tp48,$acc8 1044 shr \$7,$tp40 1045 lea ($tp10,$tp10),$tp20 1046 shr \$7,$tp48 1047 lea ($tp18,$tp18),$tp28 1048 sub $tp40,$acc0 1049 sub $tp48,$acc8 1050 and $maskfe,$tp20 1051 and $maskfe,$tp28 1052 and $mask1b,$acc0 1053 and $mask1b,$acc8 1054 xor $acc0,$tp20 1055 xor $acc8,$tp28 1056 mov $mask80,$tp80 1057 mov $mask80,$tp88 1058 1059 and $tp20,$tp80 1060 and $tp28,$tp88 1061 mov $tp80,$acc0 1062 mov $tp88,$acc8 1063 shr \$7,$tp80 1064 lea ($tp20,$tp20),$tp40 1065 shr \$7,$tp88 1066 lea ($tp28,$tp28),$tp48 1067 sub $tp80,$acc0 1068 sub $tp88,$acc8 1069 and $maskfe,$tp40 1070 and $maskfe,$tp48 1071 and $mask1b,$acc0 1072 and $mask1b,$acc8 1073 xor $acc0,$tp40 1074 xor $acc8,$tp48 1075 mov $mask80,$tp80 1076 mov $mask80,$tp88 1077 1078 and $tp40,$tp80 1079 and $tp48,$tp88 1080 mov $tp80,$acc0 1081 mov $tp88,$acc8 1082 shr \$7,$tp80 1083 xor $tp10,$tp20 # tp2^=tp1 1084 shr \$7,$tp88 1085 xor $tp18,$tp28 # tp2^=tp1 1086 sub $tp80,$acc0 1087 sub $tp88,$acc8 1088 lea ($tp40,$tp40),$tp80 1089 lea ($tp48,$tp48),$tp88 1090 xor $tp10,$tp40 # tp4^=tp1 1091 xor $tp18,$tp48 # tp4^=tp1 1092 and $maskfe,$tp80 1093 and $maskfe,$tp88 1094 and $mask1b,$acc0 1095 and $mask1b,$acc8 1096 xor $acc0,$tp80 1097 xor $acc8,$tp88 1098 1099 xor $tp80,$tp10 # tp1^=tp8 1100 xor $tp88,$tp18 # tp1^=tp8 1101 xor $tp80,$tp20 # tp2^tp1^=tp8 1102 xor $tp88,$tp28 # tp2^tp1^=tp8 1103 mov $tp10,$acc0 1104 mov $tp18,$acc8 1105 xor $tp80,$tp40 # tp4^tp1^=tp8 1106 shr \$32,$acc0 1107 xor $tp88,$tp48 # tp4^tp1^=tp8 1108 shr \$32,$acc8 1109 xor $tp20,$tp80 # tp8^=tp8^tp2^tp1=tp2^tp1 1110 rol \$8,`&LO("$tp10")` # ROTATE(tp1^tp8,8) 1111 xor $tp28,$tp88 # tp8^=tp8^tp2^tp1=tp2^tp1 1112 rol \$8,`&LO("$tp18")` # ROTATE(tp1^tp8,8) 1113 xor $tp40,$tp80 # tp2^tp1^=tp8^tp4^tp1=tp8^tp4^tp2 1114 rol \$8,`&LO("$acc0")` # ROTATE(tp1^tp8,8) 1115 xor $tp48,$tp88 # tp2^tp1^=tp8^tp4^tp1=tp8^tp4^tp2 1116 1117 rol \$8,`&LO("$acc8")` # ROTATE(tp1^tp8,8) 1118 xor `&LO("$tp80")`,`&LO("$tp10")` 1119 shr \$32,$tp80 1120 xor `&LO("$tp88")`,`&LO("$tp18")` 1121 shr \$32,$tp88 1122 xor `&LO("$tp80")`,`&LO("$acc0")` 1123 xor `&LO("$tp88")`,`&LO("$acc8")` 1124 1125 mov $tp20,$tp80 1126 rol \$24,`&LO("$tp20")` # ROTATE(tp2^tp1^tp8,24) 1127 mov $tp28,$tp88 1128 rol \$24,`&LO("$tp28")` # ROTATE(tp2^tp1^tp8,24) 1129 shr \$32,$tp80 1130 xor `&LO("$tp20")`,`&LO("$tp10")` 1131 shr \$32,$tp88 1132 xor `&LO("$tp28")`,`&LO("$tp18")` 1133 rol \$24,`&LO("$tp80")` # ROTATE(tp2^tp1^tp8,24) 1134 mov $tp40,$tp20 1135 rol \$24,`&LO("$tp88")` # ROTATE(tp2^tp1^tp8,24) 1136 mov $tp48,$tp28 1137 shr \$32,$tp20 1138 xor `&LO("$tp80")`,`&LO("$acc0")` 1139 shr \$32,$tp28 1140 xor `&LO("$tp88")`,`&LO("$acc8")` 1141 1142 `"mov 0($sbox),$mask80" if ($prefetch)` 1143 rol \$16,`&LO("$tp40")` # ROTATE(tp4^tp1^tp8,16) 1144 `"mov 64($sbox),$maskfe" if ($prefetch)` 1145 rol \$16,`&LO("$tp48")` # ROTATE(tp4^tp1^tp8,16) 1146 `"mov 128($sbox),$mask1b" if ($prefetch)` 1147 rol \$16,`&LO("$tp20")` # ROTATE(tp4^tp1^tp8,16) 1148 `"mov 192($sbox),$tp80" if ($prefetch)` 1149 xor `&LO("$tp40")`,`&LO("$tp10")` 1150 rol \$16,`&LO("$tp28")` # ROTATE(tp4^tp1^tp8,16) 1151 xor `&LO("$tp48")`,`&LO("$tp18")` 1152 `"mov 256($sbox),$tp88" if ($prefetch)` 1153 xor `&LO("$tp20")`,`&LO("$acc0")` 1154 xor `&LO("$tp28")`,`&LO("$acc8")` 1155___ 1156} 1157 1158$code.=<<___; 1159.type _x86_64_AES_decrypt_compact,\@abi-omnipotent 1160.align 16 1161_x86_64_AES_decrypt_compact: 1162 lea 128($sbox),$inp # size optimization 1163 mov 0-128($inp),$acc1 # prefetch Td4 1164 mov 32-128($inp),$acc2 1165 mov 64-128($inp),$t0 1166 mov 96-128($inp),$t1 1167 mov 128-128($inp),$acc1 1168 mov 160-128($inp),$acc2 1169 mov 192-128($inp),$t0 1170 mov 224-128($inp),$t1 1171 jmp .Ldec_loop_compact 1172 1173.align 16 1174.Ldec_loop_compact: 1175 xor 0($key),$s0 # xor with key 1176 xor 4($key),$s1 1177 xor 8($key),$s2 1178 xor 12($key),$s3 1179 lea 16($key),$key 1180___ 1181 &deccompactvert(); 1182$code.=<<___; 1183 cmp 16(%rsp),$key 1184 je .Ldec_compact_done 1185 1186 mov 256+0($sbox),$mask80 1187 shl \$32,%rbx 1188 shl \$32,%rdx 1189 mov 256+8($sbox),$maskfe 1190 or %rbx,%rax 1191 or %rdx,%rcx 1192 mov 256+16($sbox),$mask1b 1193___ 1194 &dectransform(1); 1195$code.=<<___; 1196 jmp .Ldec_loop_compact 1197.align 16 1198.Ldec_compact_done: 1199 xor 0($key),$s0 1200 xor 4($key),$s1 1201 xor 8($key),$s2 1202 xor 12($key),$s3 1203 .byte 0xf3,0xc3 # rep ret 1204.size _x86_64_AES_decrypt_compact,.-_x86_64_AES_decrypt_compact 1205___ 1206 1207# void aes_nohw_decrypt (const void *inp,void *out,const AES_KEY *key); 1208$code.=<<___; 1209.align 16 1210.globl aes_nohw_decrypt 1211.type aes_nohw_decrypt,\@function,3 1212.hidden aes_nohw_decrypt 1213aes_nohw_decrypt: 1214.cfi_startproc 1215 mov %rsp,%rax 1216.cfi_def_cfa_register %rax 1217 push %rbx 1218.cfi_push %rbx 1219 push %rbp 1220.cfi_push %rbp 1221 push %r12 1222.cfi_push %r12 1223 push %r13 1224.cfi_push %r13 1225 push %r14 1226.cfi_push %r14 1227 push %r15 1228.cfi_push %r15 1229 1230 # allocate frame "above" key schedule 1231 lea -63(%rdx),%rcx # %rdx is key argument 1232 and \$-64,%rsp 1233 sub %rsp,%rcx 1234 neg %rcx 1235 and \$0x3c0,%rcx 1236 sub %rcx,%rsp 1237 sub \$32,%rsp 1238 1239 mov %rsi,16(%rsp) # save out 1240 mov %rax,24(%rsp) # save original stack pointer 1241.cfi_cfa_expression %rsp+24,deref,+8 1242.Ldec_prologue: 1243 1244 mov %rdx,$key 1245 mov 240($key),$rnds # load rounds 1246 1247 mov 0(%rdi),$s0 # load input vector 1248 mov 4(%rdi),$s1 1249 mov 8(%rdi),$s2 1250 mov 12(%rdi),$s3 1251 1252 shl \$4,$rnds 1253 lea ($key,$rnds),%rbp 1254 mov $key,(%rsp) # key schedule 1255 mov %rbp,8(%rsp) # end of key schedule 1256 1257 # pick Td4 copy which can't "overlap" with stack frame or key schedule 1258 lea .LAES_Td+2048(%rip),$sbox 1259 lea 768(%rsp),%rbp 1260 sub $sbox,%rbp 1261 and \$0x300,%rbp 1262 lea ($sbox,%rbp),$sbox 1263 shr \$3,%rbp # recall "magic" constants! 1264 add %rbp,$sbox 1265 1266 call _x86_64_AES_decrypt_compact 1267 1268 mov 16(%rsp),$out # restore out 1269 mov 24(%rsp),%rsi # restore saved stack pointer 1270.cfi_def_cfa %rsi,8 1271 mov $s0,0($out) # write output vector 1272 mov $s1,4($out) 1273 mov $s2,8($out) 1274 mov $s3,12($out) 1275 1276 mov -48(%rsi),%r15 1277.cfi_restore %r15 1278 mov -40(%rsi),%r14 1279.cfi_restore %r14 1280 mov -32(%rsi),%r13 1281.cfi_restore %r13 1282 mov -24(%rsi),%r12 1283.cfi_restore %r12 1284 mov -16(%rsi),%rbp 1285.cfi_restore %rbp 1286 mov -8(%rsi),%rbx 1287.cfi_restore %rbx 1288 lea (%rsi),%rsp 1289.cfi_def_cfa_register %rsp 1290.Ldec_epilogue: 1291 ret 1292.cfi_endproc 1293.size aes_nohw_decrypt,.-aes_nohw_decrypt 1294___ 1295#------------------------------------------------------------------# 1296 1297sub enckey() 1298{ 1299$code.=<<___; 1300 movz %dl,%esi # rk[i]>>0 1301 movzb -128(%rbp,%rsi),%ebx 1302 movz %dh,%esi # rk[i]>>8 1303 shl \$24,%ebx 1304 xor %ebx,%eax 1305 1306 movzb -128(%rbp,%rsi),%ebx 1307 shr \$16,%edx 1308 movz %dl,%esi # rk[i]>>16 1309 xor %ebx,%eax 1310 1311 movzb -128(%rbp,%rsi),%ebx 1312 movz %dh,%esi # rk[i]>>24 1313 shl \$8,%ebx 1314 xor %ebx,%eax 1315 1316 movzb -128(%rbp,%rsi),%ebx 1317 shl \$16,%ebx 1318 xor %ebx,%eax 1319 1320 xor 1024-128(%rbp,%rcx,4),%eax # rcon 1321___ 1322} 1323 1324# int aes_nohw_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key) 1325$code.=<<___; 1326.align 16 1327.globl aes_nohw_set_encrypt_key 1328.type aes_nohw_set_encrypt_key,\@function,3 1329aes_nohw_set_encrypt_key: 1330.cfi_startproc 1331 push %rbx 1332.cfi_push %rbx 1333 push %rbp 1334.cfi_push %rbp 1335 push %r12 # redundant, but allows to share 1336.cfi_push %r12 1337 push %r13 # exception handler... 1338.cfi_push %r13 1339 push %r14 1340.cfi_push %r14 1341 push %r15 1342.cfi_push %r15 1343 sub \$8,%rsp 1344.cfi_adjust_cfa_offset 8 1345.Lenc_key_prologue: 1346 1347 call _x86_64_AES_set_encrypt_key 1348 1349 mov 40(%rsp),%rbp 1350.cfi_restore %rbp 1351 mov 48(%rsp),%rbx 1352.cfi_restore %rbx 1353 add \$56,%rsp 1354.cfi_adjust_cfa_offset -56 1355.Lenc_key_epilogue: 1356 ret 1357.cfi_endproc 1358.size aes_nohw_set_encrypt_key,.-aes_nohw_set_encrypt_key 1359 1360.type _x86_64_AES_set_encrypt_key,\@abi-omnipotent 1361.align 16 1362_x86_64_AES_set_encrypt_key: 1363 mov %esi,%ecx # %ecx=bits 1364 mov %rdi,%rsi # %rsi=userKey 1365 mov %rdx,%rdi # %rdi=key 1366 1367 test \$-1,%rsi 1368 jz .Lbadpointer 1369 test \$-1,%rdi 1370 jz .Lbadpointer 1371 1372 lea .LAES_Te(%rip),%rbp 1373 lea 2048+128(%rbp),%rbp 1374 1375 # prefetch Te4 1376 mov 0-128(%rbp),%eax 1377 mov 32-128(%rbp),%ebx 1378 mov 64-128(%rbp),%r8d 1379 mov 96-128(%rbp),%edx 1380 mov 128-128(%rbp),%eax 1381 mov 160-128(%rbp),%ebx 1382 mov 192-128(%rbp),%r8d 1383 mov 224-128(%rbp),%edx 1384 1385 cmp \$128,%ecx 1386 je .L10rounds 1387 cmp \$192,%ecx 1388 je .L12rounds 1389 cmp \$256,%ecx 1390 je .L14rounds 1391 mov \$-2,%rax # invalid number of bits 1392 jmp .Lexit 1393 1394.L10rounds: 1395 mov 0(%rsi),%rax # copy first 4 dwords 1396 mov 8(%rsi),%rdx 1397 mov %rax,0(%rdi) 1398 mov %rdx,8(%rdi) 1399 1400 shr \$32,%rdx 1401 xor %ecx,%ecx 1402 jmp .L10shortcut 1403.align 4 1404.L10loop: 1405 mov 0(%rdi),%eax # rk[0] 1406 mov 12(%rdi),%edx # rk[3] 1407.L10shortcut: 1408___ 1409 &enckey (); 1410$code.=<<___; 1411 mov %eax,16(%rdi) # rk[4] 1412 xor 4(%rdi),%eax 1413 mov %eax,20(%rdi) # rk[5] 1414 xor 8(%rdi),%eax 1415 mov %eax,24(%rdi) # rk[6] 1416 xor 12(%rdi),%eax 1417 mov %eax,28(%rdi) # rk[7] 1418 add \$1,%ecx 1419 lea 16(%rdi),%rdi 1420 cmp \$10,%ecx 1421 jl .L10loop 1422 1423 movl \$10,80(%rdi) # setup number of rounds 1424 xor %rax,%rax 1425 jmp .Lexit 1426 1427.L12rounds: 1428 mov 0(%rsi),%rax # copy first 6 dwords 1429 mov 8(%rsi),%rbx 1430 mov 16(%rsi),%rdx 1431 mov %rax,0(%rdi) 1432 mov %rbx,8(%rdi) 1433 mov %rdx,16(%rdi) 1434 1435 shr \$32,%rdx 1436 xor %ecx,%ecx 1437 jmp .L12shortcut 1438.align 4 1439.L12loop: 1440 mov 0(%rdi),%eax # rk[0] 1441 mov 20(%rdi),%edx # rk[5] 1442.L12shortcut: 1443___ 1444 &enckey (); 1445$code.=<<___; 1446 mov %eax,24(%rdi) # rk[6] 1447 xor 4(%rdi),%eax 1448 mov %eax,28(%rdi) # rk[7] 1449 xor 8(%rdi),%eax 1450 mov %eax,32(%rdi) # rk[8] 1451 xor 12(%rdi),%eax 1452 mov %eax,36(%rdi) # rk[9] 1453 1454 cmp \$7,%ecx 1455 je .L12break 1456 add \$1,%ecx 1457 1458 xor 16(%rdi),%eax 1459 mov %eax,40(%rdi) # rk[10] 1460 xor 20(%rdi),%eax 1461 mov %eax,44(%rdi) # rk[11] 1462 1463 lea 24(%rdi),%rdi 1464 jmp .L12loop 1465.L12break: 1466 movl \$12,72(%rdi) # setup number of rounds 1467 xor %rax,%rax 1468 jmp .Lexit 1469 1470.L14rounds: 1471 mov 0(%rsi),%rax # copy first 8 dwords 1472 mov 8(%rsi),%rbx 1473 mov 16(%rsi),%rcx 1474 mov 24(%rsi),%rdx 1475 mov %rax,0(%rdi) 1476 mov %rbx,8(%rdi) 1477 mov %rcx,16(%rdi) 1478 mov %rdx,24(%rdi) 1479 1480 shr \$32,%rdx 1481 xor %ecx,%ecx 1482 jmp .L14shortcut 1483.align 4 1484.L14loop: 1485 mov 0(%rdi),%eax # rk[0] 1486 mov 28(%rdi),%edx # rk[4] 1487.L14shortcut: 1488___ 1489 &enckey (); 1490$code.=<<___; 1491 mov %eax,32(%rdi) # rk[8] 1492 xor 4(%rdi),%eax 1493 mov %eax,36(%rdi) # rk[9] 1494 xor 8(%rdi),%eax 1495 mov %eax,40(%rdi) # rk[10] 1496 xor 12(%rdi),%eax 1497 mov %eax,44(%rdi) # rk[11] 1498 1499 cmp \$6,%ecx 1500 je .L14break 1501 add \$1,%ecx 1502 1503 mov %eax,%edx 1504 mov 16(%rdi),%eax # rk[4] 1505 movz %dl,%esi # rk[11]>>0 1506 movzb -128(%rbp,%rsi),%ebx 1507 movz %dh,%esi # rk[11]>>8 1508 xor %ebx,%eax 1509 1510 movzb -128(%rbp,%rsi),%ebx 1511 shr \$16,%edx 1512 shl \$8,%ebx 1513 movz %dl,%esi # rk[11]>>16 1514 xor %ebx,%eax 1515 1516 movzb -128(%rbp,%rsi),%ebx 1517 movz %dh,%esi # rk[11]>>24 1518 shl \$16,%ebx 1519 xor %ebx,%eax 1520 1521 movzb -128(%rbp,%rsi),%ebx 1522 shl \$24,%ebx 1523 xor %ebx,%eax 1524 1525 mov %eax,48(%rdi) # rk[12] 1526 xor 20(%rdi),%eax 1527 mov %eax,52(%rdi) # rk[13] 1528 xor 24(%rdi),%eax 1529 mov %eax,56(%rdi) # rk[14] 1530 xor 28(%rdi),%eax 1531 mov %eax,60(%rdi) # rk[15] 1532 1533 lea 32(%rdi),%rdi 1534 jmp .L14loop 1535.L14break: 1536 movl \$14,48(%rdi) # setup number of rounds 1537 xor %rax,%rax 1538 jmp .Lexit 1539 1540.Lbadpointer: 1541 mov \$-1,%rax 1542.Lexit: 1543 .byte 0xf3,0xc3 # rep ret 1544.size _x86_64_AES_set_encrypt_key,.-_x86_64_AES_set_encrypt_key 1545___ 1546 1547sub deckey_ref() 1548{ my ($i,$ptr,$te,$td) = @_; 1549 my ($tp1,$tp2,$tp4,$tp8,$acc)=("%eax","%ebx","%edi","%edx","%r8d"); 1550$code.=<<___; 1551 mov $i($ptr),$tp1 1552 mov $tp1,$acc 1553 and \$0x80808080,$acc 1554 mov $acc,$tp4 1555 shr \$7,$tp4 1556 lea 0($tp1,$tp1),$tp2 1557 sub $tp4,$acc 1558 and \$0xfefefefe,$tp2 1559 and \$0x1b1b1b1b,$acc 1560 xor $tp2,$acc 1561 mov $acc,$tp2 1562 1563 and \$0x80808080,$acc 1564 mov $acc,$tp8 1565 shr \$7,$tp8 1566 lea 0($tp2,$tp2),$tp4 1567 sub $tp8,$acc 1568 and \$0xfefefefe,$tp4 1569 and \$0x1b1b1b1b,$acc 1570 xor $tp1,$tp2 # tp2^tp1 1571 xor $tp4,$acc 1572 mov $acc,$tp4 1573 1574 and \$0x80808080,$acc 1575 mov $acc,$tp8 1576 shr \$7,$tp8 1577 sub $tp8,$acc 1578 lea 0($tp4,$tp4),$tp8 1579 xor $tp1,$tp4 # tp4^tp1 1580 and \$0xfefefefe,$tp8 1581 and \$0x1b1b1b1b,$acc 1582 xor $acc,$tp8 1583 1584 xor $tp8,$tp1 # tp1^tp8 1585 rol \$8,$tp1 # ROTATE(tp1^tp8,8) 1586 xor $tp8,$tp2 # tp2^tp1^tp8 1587 xor $tp8,$tp4 # tp4^tp1^tp8 1588 xor $tp2,$tp8 1589 xor $tp4,$tp8 # tp8^(tp8^tp4^tp1)^(tp8^tp2^tp1)=tp8^tp4^tp2 1590 1591 xor $tp8,$tp1 1592 rol \$24,$tp2 # ROTATE(tp2^tp1^tp8,24) 1593 xor $tp2,$tp1 1594 rol \$16,$tp4 # ROTATE(tp4^tp1^tp8,16) 1595 xor $tp4,$tp1 1596 1597 mov $tp1,$i($ptr) 1598___ 1599} 1600 1601# int aes_nohw_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key) 1602$code.=<<___; 1603.align 16 1604.globl aes_nohw_set_decrypt_key 1605.type aes_nohw_set_decrypt_key,\@function,3 1606aes_nohw_set_decrypt_key: 1607.cfi_startproc 1608 push %rbx 1609.cfi_push %rbx 1610 push %rbp 1611.cfi_push %rbp 1612 push %r12 1613.cfi_push %r12 1614 push %r13 1615.cfi_push %r13 1616 push %r14 1617.cfi_push %r14 1618 push %r15 1619.cfi_push %r15 1620 push %rdx # save key schedule 1621.cfi_adjust_cfa_offset 8 1622.Ldec_key_prologue: 1623 1624 call _x86_64_AES_set_encrypt_key 1625 mov (%rsp),%r8 # restore key schedule 1626 cmp \$0,%eax 1627 jne .Labort 1628 1629 mov 240(%r8),%r14d # pull number of rounds 1630 xor %rdi,%rdi 1631 lea (%rdi,%r14d,4),%rcx 1632 mov %r8,%rsi 1633 lea (%r8,%rcx,4),%rdi # pointer to last chunk 1634.align 4 1635.Linvert: 1636 mov 0(%rsi),%rax 1637 mov 8(%rsi),%rbx 1638 mov 0(%rdi),%rcx 1639 mov 8(%rdi),%rdx 1640 mov %rax,0(%rdi) 1641 mov %rbx,8(%rdi) 1642 mov %rcx,0(%rsi) 1643 mov %rdx,8(%rsi) 1644 lea 16(%rsi),%rsi 1645 lea -16(%rdi),%rdi 1646 cmp %rsi,%rdi 1647 jne .Linvert 1648 1649 lea .LAES_Te+2048+1024(%rip),%rax # rcon 1650 1651 mov 40(%rax),$mask80 1652 mov 48(%rax),$maskfe 1653 mov 56(%rax),$mask1b 1654 1655 mov %r8,$key 1656 sub \$1,%r14d 1657.align 4 1658.Lpermute: 1659 lea 16($key),$key 1660 mov 0($key),%rax 1661 mov 8($key),%rcx 1662___ 1663 &dectransform (); 1664$code.=<<___; 1665 mov %eax,0($key) 1666 mov %ebx,4($key) 1667 mov %ecx,8($key) 1668 mov %edx,12($key) 1669 sub \$1,%r14d 1670 jnz .Lpermute 1671 1672 xor %rax,%rax 1673.Labort: 1674 mov 8(%rsp),%r15 1675.cfi_restore %r15 1676 mov 16(%rsp),%r14 1677.cfi_restore %r14 1678 mov 24(%rsp),%r13 1679.cfi_restore %r13 1680 mov 32(%rsp),%r12 1681.cfi_restore %r12 1682 mov 40(%rsp),%rbp 1683.cfi_restore %rbp 1684 mov 48(%rsp),%rbx 1685.cfi_restore %rbx 1686 add \$56,%rsp 1687.cfi_adjust_cfa_offset -56 1688.Ldec_key_epilogue: 1689 ret 1690.cfi_endproc 1691.size aes_nohw_set_decrypt_key,.-aes_nohw_set_decrypt_key 1692___ 1693 1694# void aes_nohw_cbc_encrypt (const void char *inp, unsigned char *out, 1695# size_t length, const AES_KEY *key, 1696# unsigned char *ivp,const int enc); 1697{ 1698# stack frame layout 1699# -8(%rsp) return address 1700my $keyp="0(%rsp)"; # one to pass as $key 1701my $keyend="8(%rsp)"; # &(keyp->rd_key[4*keyp->rounds]) 1702my $_rsp="16(%rsp)"; # saved %rsp 1703my $_inp="24(%rsp)"; # copy of 1st parameter, inp 1704my $_out="32(%rsp)"; # copy of 2nd parameter, out 1705my $_len="40(%rsp)"; # copy of 3rd parameter, length 1706my $_key="48(%rsp)"; # copy of 4th parameter, key 1707my $_ivp="56(%rsp)"; # copy of 5th parameter, ivp 1708my $ivec="64(%rsp)"; # ivec[16] 1709my $aes_key="80(%rsp)"; # copy of aes_key 1710my $mark="80+240(%rsp)"; # copy of aes_key->rounds 1711 1712$code.=<<___; 1713.align 16 1714.globl aes_nohw_cbc_encrypt 1715.type aes_nohw_cbc_encrypt,\@function,6 1716.extern OPENSSL_ia32cap_P 1717.hidden aes_nohw_cbc_encrypt 1718aes_nohw_cbc_encrypt: 1719.cfi_startproc 1720 cmp \$0,%rdx # check length 1721 je .Lcbc_epilogue 1722 pushfq 1723.cfi_push 49 # %rflags 1724 push %rbx 1725.cfi_push %rbx 1726 push %rbp 1727.cfi_push %rbp 1728 push %r12 1729.cfi_push %r12 1730 push %r13 1731.cfi_push %r13 1732 push %r14 1733.cfi_push %r14 1734 push %r15 1735.cfi_push %r15 1736.Lcbc_prologue: 1737 1738 cld 1739 mov %r9d,%r9d # clear upper half of enc 1740 1741 lea .LAES_Te(%rip),$sbox 1742 lea .LAES_Td(%rip),%r10 1743 cmp \$0,%r9 1744 cmoveq %r10,$sbox 1745 1746 leaq OPENSSL_ia32cap_P(%rip),%r10 1747 mov (%r10), %r10d 1748 cmp \$$speed_limit,%rdx 1749 jb .Lcbc_slow_prologue 1750 test \$15,%rdx 1751 jnz .Lcbc_slow_prologue 1752 bt \$28,%r10d 1753 jc .Lcbc_slow_prologue 1754 1755 # allocate aligned stack frame... 1756 lea -88-248(%rsp),$key 1757 and \$-64,$key 1758 1759 # ... and make sure it doesn't alias with AES_T[ed] modulo 4096 1760 mov $sbox,%r10 1761 lea 2304($sbox),%r11 1762 mov $key,%r12 1763 and \$0xFFF,%r10 # s = $sbox&0xfff 1764 and \$0xFFF,%r11 # e = ($sbox+2048)&0xfff 1765 and \$0xFFF,%r12 # p = %rsp&0xfff 1766 1767 cmp %r11,%r12 # if (p=>e) %rsp =- (p-e); 1768 jb .Lcbc_te_break_out 1769 sub %r11,%r12 1770 sub %r12,$key 1771 jmp .Lcbc_te_ok 1772.Lcbc_te_break_out: # else %rsp -= (p-s)&0xfff + framesz 1773 sub %r10,%r12 1774 and \$0xFFF,%r12 1775 add \$320,%r12 1776 sub %r12,$key 1777.align 4 1778.Lcbc_te_ok: 1779 1780 xchg %rsp,$key 1781.cfi_def_cfa_register $key 1782 #add \$8,%rsp # reserve for return address! 1783 mov $key,$_rsp # save %rsp 1784.cfi_cfa_expression $_rsp,deref,+64 1785.Lcbc_fast_body: 1786 mov %rdi,$_inp # save copy of inp 1787 mov %rsi,$_out # save copy of out 1788 mov %rdx,$_len # save copy of len 1789 mov %rcx,$_key # save copy of key 1790 mov %r8,$_ivp # save copy of ivp 1791 movl \$0,$mark # copy of aes_key->rounds = 0; 1792 mov %r8,%rbp # rearrange input arguments 1793 mov %r9,%rbx 1794 mov %rsi,$out 1795 mov %rdi,$inp 1796 mov %rcx,$key 1797 1798 mov 240($key),%eax # key->rounds 1799 # do we copy key schedule to stack? 1800 mov $key,%r10 1801 sub $sbox,%r10 1802 and \$0xfff,%r10 1803 cmp \$2304,%r10 1804 jb .Lcbc_do_ecopy 1805 cmp \$4096-248,%r10 1806 jb .Lcbc_skip_ecopy 1807.align 4 1808.Lcbc_do_ecopy: 1809 mov $key,%rsi 1810 lea $aes_key,%rdi 1811 lea $aes_key,$key 1812 mov \$240/8,%ecx 1813 .long 0x90A548F3 # rep movsq 1814 mov %eax,(%rdi) # copy aes_key->rounds 1815.Lcbc_skip_ecopy: 1816 mov $key,$keyp # save key pointer 1817 1818 mov \$18,%ecx 1819.align 4 1820.Lcbc_prefetch_te: 1821 mov 0($sbox),%r10 1822 mov 32($sbox),%r11 1823 mov 64($sbox),%r12 1824 mov 96($sbox),%r13 1825 lea 128($sbox),$sbox 1826 sub \$1,%ecx 1827 jnz .Lcbc_prefetch_te 1828 lea -2304($sbox),$sbox 1829 1830 cmp \$0,%rbx 1831 je .LFAST_DECRYPT 1832 1833#----------------------------- ENCRYPT -----------------------------# 1834 mov 0(%rbp),$s0 # load iv 1835 mov 4(%rbp),$s1 1836 mov 8(%rbp),$s2 1837 mov 12(%rbp),$s3 1838 1839.align 4 1840.Lcbc_fast_enc_loop: 1841 xor 0($inp),$s0 1842 xor 4($inp),$s1 1843 xor 8($inp),$s2 1844 xor 12($inp),$s3 1845 mov $keyp,$key # restore key 1846 mov $inp,$_inp # if ($verticalspin) save inp 1847 1848 call _x86_64_AES_encrypt 1849 1850 mov $_inp,$inp # if ($verticalspin) restore inp 1851 mov $_len,%r10 1852 mov $s0,0($out) 1853 mov $s1,4($out) 1854 mov $s2,8($out) 1855 mov $s3,12($out) 1856 1857 lea 16($inp),$inp 1858 lea 16($out),$out 1859 sub \$16,%r10 1860 test \$-16,%r10 1861 mov %r10,$_len 1862 jnz .Lcbc_fast_enc_loop 1863 mov $_ivp,%rbp # restore ivp 1864 mov $s0,0(%rbp) # save ivec 1865 mov $s1,4(%rbp) 1866 mov $s2,8(%rbp) 1867 mov $s3,12(%rbp) 1868 1869 jmp .Lcbc_fast_cleanup 1870 1871#----------------------------- DECRYPT -----------------------------# 1872.align 16 1873.LFAST_DECRYPT: 1874 cmp $inp,$out 1875 je .Lcbc_fast_dec_in_place 1876 1877 mov %rbp,$ivec 1878.align 4 1879.Lcbc_fast_dec_loop: 1880 mov 0($inp),$s0 # read input 1881 mov 4($inp),$s1 1882 mov 8($inp),$s2 1883 mov 12($inp),$s3 1884 mov $keyp,$key # restore key 1885 mov $inp,$_inp # if ($verticalspin) save inp 1886 1887 call _x86_64_AES_decrypt 1888 1889 mov $ivec,%rbp # load ivp 1890 mov $_inp,$inp # if ($verticalspin) restore inp 1891 mov $_len,%r10 # load len 1892 xor 0(%rbp),$s0 # xor iv 1893 xor 4(%rbp),$s1 1894 xor 8(%rbp),$s2 1895 xor 12(%rbp),$s3 1896 mov $inp,%rbp # current input, next iv 1897 1898 sub \$16,%r10 1899 mov %r10,$_len # update len 1900 mov %rbp,$ivec # update ivp 1901 1902 mov $s0,0($out) # write output 1903 mov $s1,4($out) 1904 mov $s2,8($out) 1905 mov $s3,12($out) 1906 1907 lea 16($inp),$inp 1908 lea 16($out),$out 1909 jnz .Lcbc_fast_dec_loop 1910 mov $_ivp,%r12 # load user ivp 1911 mov 0(%rbp),%r10 # load iv 1912 mov 8(%rbp),%r11 1913 mov %r10,0(%r12) # copy back to user 1914 mov %r11,8(%r12) 1915 jmp .Lcbc_fast_cleanup 1916 1917.align 16 1918.Lcbc_fast_dec_in_place: 1919 mov 0(%rbp),%r10 # copy iv to stack 1920 mov 8(%rbp),%r11 1921 mov %r10,0+$ivec 1922 mov %r11,8+$ivec 1923.align 4 1924.Lcbc_fast_dec_in_place_loop: 1925 mov 0($inp),$s0 # load input 1926 mov 4($inp),$s1 1927 mov 8($inp),$s2 1928 mov 12($inp),$s3 1929 mov $keyp,$key # restore key 1930 mov $inp,$_inp # if ($verticalspin) save inp 1931 1932 call _x86_64_AES_decrypt 1933 1934 mov $_inp,$inp # if ($verticalspin) restore inp 1935 mov $_len,%r10 1936 xor 0+$ivec,$s0 1937 xor 4+$ivec,$s1 1938 xor 8+$ivec,$s2 1939 xor 12+$ivec,$s3 1940 1941 mov 0($inp),%r11 # load input 1942 mov 8($inp),%r12 1943 sub \$16,%r10 1944 jz .Lcbc_fast_dec_in_place_done 1945 1946 mov %r11,0+$ivec # copy input to iv 1947 mov %r12,8+$ivec 1948 1949 mov $s0,0($out) # save output [zaps input] 1950 mov $s1,4($out) 1951 mov $s2,8($out) 1952 mov $s3,12($out) 1953 1954 lea 16($inp),$inp 1955 lea 16($out),$out 1956 mov %r10,$_len 1957 jmp .Lcbc_fast_dec_in_place_loop 1958.Lcbc_fast_dec_in_place_done: 1959 mov $_ivp,%rdi 1960 mov %r11,0(%rdi) # copy iv back to user 1961 mov %r12,8(%rdi) 1962 1963 mov $s0,0($out) # save output [zaps input] 1964 mov $s1,4($out) 1965 mov $s2,8($out) 1966 mov $s3,12($out) 1967 1968.align 4 1969.Lcbc_fast_cleanup: 1970 cmpl \$0,$mark # was the key schedule copied? 1971 lea $aes_key,%rdi 1972 je .Lcbc_exit 1973 mov \$240/8,%ecx 1974 xor %rax,%rax 1975 .long 0x90AB48F3 # rep stosq 1976 1977 jmp .Lcbc_exit 1978 1979#--------------------------- SLOW ROUTINE ---------------------------# 1980.align 16 1981.Lcbc_slow_prologue: 1982 # allocate aligned stack frame... 1983 lea -88(%rsp),%rbp 1984 and \$-64,%rbp 1985 # ... just "above" key schedule 1986 lea -88-63(%rcx),%r10 1987 sub %rbp,%r10 1988 neg %r10 1989 and \$0x3c0,%r10 1990 sub %r10,%rbp 1991 1992 xchg %rsp,%rbp 1993 #add \$8,%rsp # reserve for return address! 1994 mov %rbp,$_rsp # save %rsp 1995.Lcbc_slow_body: 1996 #mov %rdi,$_inp # save copy of inp 1997 #mov %rsi,$_out # save copy of out 1998 #mov %rdx,$_len # save copy of len 1999 #mov %rcx,$_key # save copy of key 2000 mov %r8,$_ivp # save copy of ivp 2001 mov %r8,%rbp # rearrange input arguments 2002 mov %r9,%rbx 2003 mov %rsi,$out 2004 mov %rdi,$inp 2005 mov %rcx,$key 2006 mov %rdx,%r10 2007 2008 mov 240($key),%eax 2009 mov $key,$keyp # save key pointer 2010 shl \$4,%eax 2011 lea ($key,%rax),%rax 2012 mov %rax,$keyend 2013 2014 # pick Te4 copy which can't "overlap" with stack frame or key schedule 2015 lea 2048($sbox),$sbox 2016 lea 768-8(%rsp),%rax 2017 sub $sbox,%rax 2018 and \$0x300,%rax 2019 lea ($sbox,%rax),$sbox 2020 2021 cmp \$0,%rbx 2022 je .LSLOW_DECRYPT 2023 2024#--------------------------- SLOW ENCRYPT ---------------------------# 2025 test \$-16,%r10 # check upon length 2026 mov 0(%rbp),$s0 # load iv 2027 mov 4(%rbp),$s1 2028 mov 8(%rbp),$s2 2029 mov 12(%rbp),$s3 2030 jz .Lcbc_slow_enc_tail # short input... 2031 2032.align 4 2033.Lcbc_slow_enc_loop: 2034 xor 0($inp),$s0 2035 xor 4($inp),$s1 2036 xor 8($inp),$s2 2037 xor 12($inp),$s3 2038 mov $keyp,$key # restore key 2039 mov $inp,$_inp # save inp 2040 mov $out,$_out # save out 2041 mov %r10,$_len # save len 2042 2043 call _x86_64_AES_encrypt_compact 2044 2045 mov $_inp,$inp # restore inp 2046 mov $_out,$out # restore out 2047 mov $_len,%r10 # restore len 2048 mov $s0,0($out) 2049 mov $s1,4($out) 2050 mov $s2,8($out) 2051 mov $s3,12($out) 2052 2053 lea 16($inp),$inp 2054 lea 16($out),$out 2055 sub \$16,%r10 2056 test \$-16,%r10 2057 jnz .Lcbc_slow_enc_loop 2058 test \$15,%r10 2059 jnz .Lcbc_slow_enc_tail 2060 mov $_ivp,%rbp # restore ivp 2061 mov $s0,0(%rbp) # save ivec 2062 mov $s1,4(%rbp) 2063 mov $s2,8(%rbp) 2064 mov $s3,12(%rbp) 2065 2066 jmp .Lcbc_exit 2067 2068.align 4 2069.Lcbc_slow_enc_tail: 2070 mov %rax,%r11 2071 mov %rcx,%r12 2072 mov %r10,%rcx 2073 mov $inp,%rsi 2074 mov $out,%rdi 2075 .long 0x9066A4F3 # rep movsb 2076 mov \$16,%rcx # zero tail 2077 sub %r10,%rcx 2078 xor %rax,%rax 2079 .long 0x9066AAF3 # rep stosb 2080 mov $out,$inp # this is not a mistake! 2081 mov \$16,%r10 # len=16 2082 mov %r11,%rax 2083 mov %r12,%rcx 2084 jmp .Lcbc_slow_enc_loop # one more spin... 2085#--------------------------- SLOW DECRYPT ---------------------------# 2086.align 16 2087.LSLOW_DECRYPT: 2088 shr \$3,%rax 2089 add %rax,$sbox # recall "magic" constants! 2090 2091 mov 0(%rbp),%r11 # copy iv to stack 2092 mov 8(%rbp),%r12 2093 mov %r11,0+$ivec 2094 mov %r12,8+$ivec 2095 2096.align 4 2097.Lcbc_slow_dec_loop: 2098 mov 0($inp),$s0 # load input 2099 mov 4($inp),$s1 2100 mov 8($inp),$s2 2101 mov 12($inp),$s3 2102 mov $keyp,$key # restore key 2103 mov $inp,$_inp # save inp 2104 mov $out,$_out # save out 2105 mov %r10,$_len # save len 2106 2107 call _x86_64_AES_decrypt_compact 2108 2109 mov $_inp,$inp # restore inp 2110 mov $_out,$out # restore out 2111 mov $_len,%r10 2112 xor 0+$ivec,$s0 2113 xor 4+$ivec,$s1 2114 xor 8+$ivec,$s2 2115 xor 12+$ivec,$s3 2116 2117 mov 0($inp),%r11 # load input 2118 mov 8($inp),%r12 2119 sub \$16,%r10 2120 jc .Lcbc_slow_dec_partial 2121 jz .Lcbc_slow_dec_done 2122 2123 mov %r11,0+$ivec # copy input to iv 2124 mov %r12,8+$ivec 2125 2126 mov $s0,0($out) # save output [can zap input] 2127 mov $s1,4($out) 2128 mov $s2,8($out) 2129 mov $s3,12($out) 2130 2131 lea 16($inp),$inp 2132 lea 16($out),$out 2133 jmp .Lcbc_slow_dec_loop 2134.Lcbc_slow_dec_done: 2135 mov $_ivp,%rdi 2136 mov %r11,0(%rdi) # copy iv back to user 2137 mov %r12,8(%rdi) 2138 2139 mov $s0,0($out) # save output [can zap input] 2140 mov $s1,4($out) 2141 mov $s2,8($out) 2142 mov $s3,12($out) 2143 2144 jmp .Lcbc_exit 2145 2146.align 4 2147.Lcbc_slow_dec_partial: 2148 mov $_ivp,%rdi 2149 mov %r11,0(%rdi) # copy iv back to user 2150 mov %r12,8(%rdi) 2151 2152 mov $s0,0+$ivec # save output to stack 2153 mov $s1,4+$ivec 2154 mov $s2,8+$ivec 2155 mov $s3,12+$ivec 2156 2157 mov $out,%rdi 2158 lea $ivec,%rsi 2159 lea 16(%r10),%rcx 2160 .long 0x9066A4F3 # rep movsb 2161 jmp .Lcbc_exit 2162 2163.align 16 2164.Lcbc_exit: 2165 mov $_rsp,%rsi 2166.cfi_def_cfa %rsi,64 2167 mov (%rsi),%r15 2168.cfi_restore %r15 2169 mov 8(%rsi),%r14 2170.cfi_restore %r14 2171 mov 16(%rsi),%r13 2172.cfi_restore %r13 2173 mov 24(%rsi),%r12 2174.cfi_restore %r12 2175 mov 32(%rsi),%rbp 2176.cfi_restore %rbp 2177 mov 40(%rsi),%rbx 2178.cfi_restore %rbx 2179 lea 48(%rsi),%rsp 2180.cfi_def_cfa %rsp,16 2181.Lcbc_popfq: 2182 popfq 2183.cfi_pop 49 # %rflags 2184.Lcbc_epilogue: 2185 ret 2186.cfi_endproc 2187.size aes_nohw_cbc_encrypt,.-aes_nohw_cbc_encrypt 2188___ 2189} 2190 2191$code.=<<___; 2192.align 64 2193.LAES_Te: 2194___ 2195 &_data_word(0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6); 2196 &_data_word(0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591); 2197 &_data_word(0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56); 2198 &_data_word(0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec); 2199 &_data_word(0x45caca8f, 0x9d82821f, 0x40c9c989, 0x877d7dfa); 2200 &_data_word(0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb); 2201 &_data_word(0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45); 2202 &_data_word(0xbf9c9c23, 0xf7a4a453, 0x967272e4, 0x5bc0c09b); 2203 &_data_word(0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c); 2204 &_data_word(0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83); 2205 &_data_word(0x5c343468, 0xf4a5a551, 0x34e5e5d1, 0x08f1f1f9); 2206 &_data_word(0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a); 2207 &_data_word(0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d); 2208 &_data_word(0x28181830, 0xa1969637, 0x0f05050a, 0xb59a9a2f); 2209 &_data_word(0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df); 2210 &_data_word(0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea); 2211 &_data_word(0x1b090912, 0x9e83831d, 0x742c2c58, 0x2e1a1a34); 2212 &_data_word(0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b); 2213 &_data_word(0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d); 2214 &_data_word(0x7b292952, 0x3ee3e3dd, 0x712f2f5e, 0x97848413); 2215 &_data_word(0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1); 2216 &_data_word(0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6); 2217 &_data_word(0xbe6a6ad4, 0x46cbcb8d, 0xd9bebe67, 0x4b393972); 2218 &_data_word(0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85); 2219 &_data_word(0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed); 2220 &_data_word(0xc5434386, 0xd74d4d9a, 0x55333366, 0x94858511); 2221 &_data_word(0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe); 2222 &_data_word(0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b); 2223 &_data_word(0xf35151a2, 0xfea3a35d, 0xc0404080, 0x8a8f8f05); 2224 &_data_word(0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1); 2225 &_data_word(0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142); 2226 &_data_word(0x30101020, 0x1affffe5, 0x0ef3f3fd, 0x6dd2d2bf); 2227 &_data_word(0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3); 2228 &_data_word(0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e); 2229 &_data_word(0x57c4c493, 0xf2a7a755, 0x827e7efc, 0x473d3d7a); 2230 &_data_word(0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6); 2231 &_data_word(0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3); 2232 &_data_word(0x66222244, 0x7e2a2a54, 0xab90903b, 0x8388880b); 2233 &_data_word(0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428); 2234 &_data_word(0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad); 2235 &_data_word(0x3be0e0db, 0x56323264, 0x4e3a3a74, 0x1e0a0a14); 2236 &_data_word(0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8); 2237 &_data_word(0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4); 2238 &_data_word(0xa8919139, 0xa4959531, 0x37e4e4d3, 0x8b7979f2); 2239 &_data_word(0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda); 2240 &_data_word(0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949); 2241 &_data_word(0xb46c6cd8, 0xfa5656ac, 0x07f4f4f3, 0x25eaeacf); 2242 &_data_word(0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810); 2243 &_data_word(0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c); 2244 &_data_word(0x241c1c38, 0xf1a6a657, 0xc7b4b473, 0x51c6c697); 2245 &_data_word(0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e); 2246 &_data_word(0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f); 2247 &_data_word(0x907070e0, 0x423e3e7c, 0xc4b5b571, 0xaa6666cc); 2248 &_data_word(0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c); 2249 &_data_word(0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969); 2250 &_data_word(0x91868617, 0x58c1c199, 0x271d1d3a, 0xb99e9e27); 2251 &_data_word(0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122); 2252 &_data_word(0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433); 2253 &_data_word(0xb69b9b2d, 0x221e1e3c, 0x92878715, 0x20e9e9c9); 2254 &_data_word(0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5); 2255 &_data_word(0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a); 2256 &_data_word(0xdabfbf65, 0x31e6e6d7, 0xc6424284, 0xb86868d0); 2257 &_data_word(0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e); 2258 &_data_word(0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c); 2259 2260#Te4 # four copies of Te4 to choose from to avoid L1 aliasing 2261 &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5); 2262 &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76); 2263 &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0); 2264 &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0); 2265 &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc); 2266 &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15); 2267 &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a); 2268 &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75); 2269 &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0); 2270 &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84); 2271 &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b); 2272 &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf); 2273 &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85); 2274 &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8); 2275 &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5); 2276 &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2); 2277 &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17); 2278 &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73); 2279 &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88); 2280 &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb); 2281 &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c); 2282 &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79); 2283 &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9); 2284 &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08); 2285 &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6); 2286 &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a); 2287 &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e); 2288 &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e); 2289 &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94); 2290 &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf); 2291 &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68); 2292 &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16); 2293 2294 &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5); 2295 &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76); 2296 &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0); 2297 &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0); 2298 &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc); 2299 &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15); 2300 &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a); 2301 &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75); 2302 &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0); 2303 &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84); 2304 &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b); 2305 &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf); 2306 &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85); 2307 &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8); 2308 &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5); 2309 &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2); 2310 &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17); 2311 &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73); 2312 &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88); 2313 &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb); 2314 &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c); 2315 &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79); 2316 &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9); 2317 &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08); 2318 &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6); 2319 &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a); 2320 &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e); 2321 &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e); 2322 &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94); 2323 &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf); 2324 &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68); 2325 &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16); 2326 2327 &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5); 2328 &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76); 2329 &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0); 2330 &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0); 2331 &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc); 2332 &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15); 2333 &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a); 2334 &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75); 2335 &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0); 2336 &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84); 2337 &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b); 2338 &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf); 2339 &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85); 2340 &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8); 2341 &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5); 2342 &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2); 2343 &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17); 2344 &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73); 2345 &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88); 2346 &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb); 2347 &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c); 2348 &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79); 2349 &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9); 2350 &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08); 2351 &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6); 2352 &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a); 2353 &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e); 2354 &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e); 2355 &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94); 2356 &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf); 2357 &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68); 2358 &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16); 2359 2360 &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5); 2361 &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76); 2362 &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0); 2363 &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0); 2364 &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc); 2365 &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15); 2366 &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a); 2367 &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75); 2368 &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0); 2369 &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84); 2370 &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b); 2371 &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf); 2372 &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85); 2373 &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8); 2374 &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5); 2375 &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2); 2376 &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17); 2377 &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73); 2378 &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88); 2379 &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb); 2380 &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c); 2381 &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79); 2382 &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9); 2383 &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08); 2384 &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6); 2385 &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a); 2386 &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e); 2387 &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e); 2388 &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94); 2389 &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf); 2390 &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68); 2391 &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16); 2392#rcon: 2393$code.=<<___; 2394 .long 0x00000001, 0x00000002, 0x00000004, 0x00000008 2395 .long 0x00000010, 0x00000020, 0x00000040, 0x00000080 2396 .long 0x0000001b, 0x00000036, 0x80808080, 0x80808080 2397 .long 0xfefefefe, 0xfefefefe, 0x1b1b1b1b, 0x1b1b1b1b 2398___ 2399$code.=<<___; 2400.align 64 2401.LAES_Td: 2402___ 2403 &_data_word(0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a); 2404 &_data_word(0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b); 2405 &_data_word(0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5); 2406 &_data_word(0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5); 2407 &_data_word(0x495ab1de, 0x671bba25, 0x980eea45, 0xe1c0fe5d); 2408 &_data_word(0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b); 2409 &_data_word(0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295); 2410 &_data_word(0x2d83bed4, 0xd3217458, 0x2969e049, 0x44c8c98e); 2411 &_data_word(0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927); 2412 &_data_word(0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d); 2413 &_data_word(0x184adf63, 0x82311ae5, 0x60335197, 0x457f5362); 2414 &_data_word(0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9); 2415 &_data_word(0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52); 2416 &_data_word(0x23d373ab, 0xe2024b72, 0x578f1fe3, 0x2aab5566); 2417 &_data_word(0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3); 2418 &_data_word(0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed); 2419 &_data_word(0x2b1ccf8a, 0x92b479a7, 0xf0f207f3, 0xa1e2694e); 2420 &_data_word(0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4); 2421 &_data_word(0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4); 2422 &_data_word(0x39ec830b, 0xaaef6040, 0x069f715e, 0x51106ebd); 2423 &_data_word(0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d); 2424 &_data_word(0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060); 2425 &_data_word(0x24fb9819, 0x97e9bdd6, 0xcc434089, 0x779ed967); 2426 &_data_word(0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879); 2427 &_data_word(0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000); 2428 &_data_word(0x83868009, 0x48ed2b32, 0xac70111e, 0x4e725a6c); 2429 &_data_word(0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36); 2430 &_data_word(0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624); 2431 &_data_word(0xb1670a0c, 0x0fe75793, 0xd296eeb4, 0x9e919b1b); 2432 &_data_word(0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c); 2433 &_data_word(0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12); 2434 &_data_word(0x0b0d090e, 0xadc78bf2, 0xb9a8b62d, 0xc8a91e14); 2435 &_data_word(0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3); 2436 &_data_word(0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b); 2437 &_data_word(0x7629438b, 0xdcc623cb, 0x68fcedb6, 0x63f1e4b8); 2438 &_data_word(0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684); 2439 &_data_word(0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7); 2440 &_data_word(0x4b2f9e1d, 0xf330b2dc, 0xec52860d, 0xd0e3c177); 2441 &_data_word(0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947); 2442 &_data_word(0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322); 2443 &_data_word(0xc74e4987, 0xc1d138d9, 0xfea2ca8c, 0x360bd498); 2444 &_data_word(0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f); 2445 &_data_word(0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54); 2446 &_data_word(0xc2138df6, 0xe8b8d890, 0x5ef7392e, 0xf5afc382); 2447 &_data_word(0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf); 2448 &_data_word(0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb); 2449 &_data_word(0x097826cd, 0xf418596e, 0x01b79aec, 0xa89a4f83); 2450 &_data_word(0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef); 2451 &_data_word(0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029); 2452 &_data_word(0xafb2a431, 0x31233f2a, 0x3094a5c6, 0xc066a235); 2453 &_data_word(0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733); 2454 &_data_word(0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117); 2455 &_data_word(0x8dd64d76, 0x4db0ef43, 0x544daacc, 0xdf0496e4); 2456 &_data_word(0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546); 2457 &_data_word(0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb); 2458 &_data_word(0x5a1d67b3, 0x52d2db92, 0x335610e9, 0x1347d66d); 2459 &_data_word(0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb); 2460 &_data_word(0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a); 2461 &_data_word(0x59dfd29c, 0x3f73f255, 0x79ce1418, 0xbf37c773); 2462 &_data_word(0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478); 2463 &_data_word(0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2); 2464 &_data_word(0x72c31d16, 0x0c25e2bc, 0x8b493c28, 0x41950dff); 2465 &_data_word(0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664); 2466 &_data_word(0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0); 2467 2468#Td4: # four copies of Td4 to choose from to avoid L1 aliasing 2469 &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38); 2470 &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb); 2471 &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87); 2472 &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb); 2473 &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d); 2474 &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e); 2475 &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2); 2476 &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25); 2477 &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16); 2478 &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92); 2479 &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda); 2480 &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84); 2481 &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a); 2482 &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06); 2483 &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02); 2484 &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b); 2485 &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea); 2486 &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73); 2487 &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85); 2488 &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e); 2489 &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89); 2490 &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b); 2491 &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20); 2492 &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4); 2493 &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31); 2494 &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f); 2495 &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d); 2496 &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef); 2497 &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0); 2498 &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61); 2499 &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26); 2500 &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d); 2501$code.=<<___; 2502 .long 0x80808080, 0x80808080, 0xfefefefe, 0xfefefefe 2503 .long 0x1b1b1b1b, 0x1b1b1b1b, 0, 0 2504___ 2505 &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38); 2506 &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb); 2507 &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87); 2508 &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb); 2509 &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d); 2510 &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e); 2511 &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2); 2512 &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25); 2513 &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16); 2514 &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92); 2515 &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda); 2516 &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84); 2517 &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a); 2518 &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06); 2519 &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02); 2520 &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b); 2521 &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea); 2522 &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73); 2523 &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85); 2524 &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e); 2525 &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89); 2526 &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b); 2527 &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20); 2528 &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4); 2529 &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31); 2530 &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f); 2531 &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d); 2532 &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef); 2533 &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0); 2534 &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61); 2535 &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26); 2536 &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d); 2537$code.=<<___; 2538 .long 0x80808080, 0x80808080, 0xfefefefe, 0xfefefefe 2539 .long 0x1b1b1b1b, 0x1b1b1b1b, 0, 0 2540___ 2541 &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38); 2542 &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb); 2543 &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87); 2544 &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb); 2545 &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d); 2546 &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e); 2547 &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2); 2548 &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25); 2549 &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16); 2550 &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92); 2551 &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda); 2552 &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84); 2553 &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a); 2554 &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06); 2555 &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02); 2556 &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b); 2557 &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea); 2558 &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73); 2559 &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85); 2560 &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e); 2561 &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89); 2562 &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b); 2563 &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20); 2564 &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4); 2565 &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31); 2566 &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f); 2567 &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d); 2568 &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef); 2569 &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0); 2570 &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61); 2571 &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26); 2572 &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d); 2573$code.=<<___; 2574 .long 0x80808080, 0x80808080, 0xfefefefe, 0xfefefefe 2575 .long 0x1b1b1b1b, 0x1b1b1b1b, 0, 0 2576___ 2577 &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38); 2578 &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb); 2579 &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87); 2580 &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb); 2581 &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d); 2582 &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e); 2583 &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2); 2584 &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25); 2585 &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16); 2586 &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92); 2587 &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda); 2588 &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84); 2589 &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a); 2590 &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06); 2591 &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02); 2592 &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b); 2593 &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea); 2594 &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73); 2595 &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85); 2596 &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e); 2597 &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89); 2598 &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b); 2599 &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20); 2600 &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4); 2601 &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31); 2602 &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f); 2603 &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d); 2604 &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef); 2605 &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0); 2606 &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61); 2607 &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26); 2608 &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d); 2609$code.=<<___; 2610 .long 0x80808080, 0x80808080, 0xfefefefe, 0xfefefefe 2611 .long 0x1b1b1b1b, 0x1b1b1b1b, 0, 0 2612.asciz "AES for x86_64, CRYPTOGAMS by <appro\@openssl.org>" 2613.align 64 2614___ 2615 2616# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 2617# CONTEXT *context,DISPATCHER_CONTEXT *disp) 2618if ($win64) { 2619$rec="%rcx"; 2620$frame="%rdx"; 2621$context="%r8"; 2622$disp="%r9"; 2623 2624$code.=<<___; 2625.extern __imp_RtlVirtualUnwind 2626.type block_se_handler,\@abi-omnipotent 2627.align 16 2628block_se_handler: 2629 push %rsi 2630 push %rdi 2631 push %rbx 2632 push %rbp 2633 push %r12 2634 push %r13 2635 push %r14 2636 push %r15 2637 pushfq 2638 sub \$64,%rsp 2639 2640 mov 120($context),%rax # pull context->Rax 2641 mov 248($context),%rbx # pull context->Rip 2642 2643 mov 8($disp),%rsi # disp->ImageBase 2644 mov 56($disp),%r11 # disp->HandlerData 2645 2646 mov 0(%r11),%r10d # HandlerData[0] 2647 lea (%rsi,%r10),%r10 # prologue label 2648 cmp %r10,%rbx # context->Rip<prologue label 2649 jb .Lin_block_prologue 2650 2651 mov 152($context),%rax # pull context->Rsp 2652 2653 mov 4(%r11),%r10d # HandlerData[1] 2654 lea (%rsi,%r10),%r10 # epilogue label 2655 cmp %r10,%rbx # context->Rip>=epilogue label 2656 jae .Lin_block_prologue 2657 2658 mov 24(%rax),%rax # pull saved real stack pointer 2659 2660 mov -8(%rax),%rbx 2661 mov -16(%rax),%rbp 2662 mov -24(%rax),%r12 2663 mov -32(%rax),%r13 2664 mov -40(%rax),%r14 2665 mov -48(%rax),%r15 2666 mov %rbx,144($context) # restore context->Rbx 2667 mov %rbp,160($context) # restore context->Rbp 2668 mov %r12,216($context) # restore context->R12 2669 mov %r13,224($context) # restore context->R13 2670 mov %r14,232($context) # restore context->R14 2671 mov %r15,240($context) # restore context->R15 2672 2673.Lin_block_prologue: 2674 mov 8(%rax),%rdi 2675 mov 16(%rax),%rsi 2676 mov %rax,152($context) # restore context->Rsp 2677 mov %rsi,168($context) # restore context->Rsi 2678 mov %rdi,176($context) # restore context->Rdi 2679 2680 jmp .Lcommon_seh_exit 2681.size block_se_handler,.-block_se_handler 2682 2683.type key_se_handler,\@abi-omnipotent 2684.align 16 2685key_se_handler: 2686 push %rsi 2687 push %rdi 2688 push %rbx 2689 push %rbp 2690 push %r12 2691 push %r13 2692 push %r14 2693 push %r15 2694 pushfq 2695 sub \$64,%rsp 2696 2697 mov 120($context),%rax # pull context->Rax 2698 mov 248($context),%rbx # pull context->Rip 2699 2700 mov 8($disp),%rsi # disp->ImageBase 2701 mov 56($disp),%r11 # disp->HandlerData 2702 2703 mov 0(%r11),%r10d # HandlerData[0] 2704 lea (%rsi,%r10),%r10 # prologue label 2705 cmp %r10,%rbx # context->Rip<prologue label 2706 jb .Lin_key_prologue 2707 2708 mov 152($context),%rax # pull context->Rsp 2709 2710 mov 4(%r11),%r10d # HandlerData[1] 2711 lea (%rsi,%r10),%r10 # epilogue label 2712 cmp %r10,%rbx # context->Rip>=epilogue label 2713 jae .Lin_key_prologue 2714 2715 lea 56(%rax),%rax 2716 2717 mov -8(%rax),%rbx 2718 mov -16(%rax),%rbp 2719 mov -24(%rax),%r12 2720 mov -32(%rax),%r13 2721 mov -40(%rax),%r14 2722 mov -48(%rax),%r15 2723 mov %rbx,144($context) # restore context->Rbx 2724 mov %rbp,160($context) # restore context->Rbp 2725 mov %r12,216($context) # restore context->R12 2726 mov %r13,224($context) # restore context->R13 2727 mov %r14,232($context) # restore context->R14 2728 mov %r15,240($context) # restore context->R15 2729 2730.Lin_key_prologue: 2731 mov 8(%rax),%rdi 2732 mov 16(%rax),%rsi 2733 mov %rax,152($context) # restore context->Rsp 2734 mov %rsi,168($context) # restore context->Rsi 2735 mov %rdi,176($context) # restore context->Rdi 2736 2737 jmp .Lcommon_seh_exit 2738.size key_se_handler,.-key_se_handler 2739 2740.type cbc_se_handler,\@abi-omnipotent 2741.align 16 2742cbc_se_handler: 2743 push %rsi 2744 push %rdi 2745 push %rbx 2746 push %rbp 2747 push %r12 2748 push %r13 2749 push %r14 2750 push %r15 2751 pushfq 2752 sub \$64,%rsp 2753 2754 mov 120($context),%rax # pull context->Rax 2755 mov 248($context),%rbx # pull context->Rip 2756 2757 lea .Lcbc_prologue(%rip),%r10 2758 cmp %r10,%rbx # context->Rip<.Lcbc_prologue 2759 jb .Lin_cbc_prologue 2760 2761 lea .Lcbc_fast_body(%rip),%r10 2762 cmp %r10,%rbx # context->Rip<.Lcbc_fast_body 2763 jb .Lin_cbc_frame_setup 2764 2765 lea .Lcbc_slow_prologue(%rip),%r10 2766 cmp %r10,%rbx # context->Rip<.Lcbc_slow_prologue 2767 jb .Lin_cbc_body 2768 2769 lea .Lcbc_slow_body(%rip),%r10 2770 cmp %r10,%rbx # context->Rip<.Lcbc_slow_body 2771 jb .Lin_cbc_frame_setup 2772 2773.Lin_cbc_body: 2774 mov 152($context),%rax # pull context->Rsp 2775 2776 lea .Lcbc_epilogue(%rip),%r10 2777 cmp %r10,%rbx # context->Rip>=.Lcbc_epilogue 2778 jae .Lin_cbc_prologue 2779 2780 lea 8(%rax),%rax 2781 2782 lea .Lcbc_popfq(%rip),%r10 2783 cmp %r10,%rbx # context->Rip>=.Lcbc_popfq 2784 jae .Lin_cbc_prologue 2785 2786 mov `16-8`(%rax),%rax # biased $_rsp 2787 lea 56(%rax),%rax 2788 2789.Lin_cbc_frame_setup: 2790 mov -16(%rax),%rbx 2791 mov -24(%rax),%rbp 2792 mov -32(%rax),%r12 2793 mov -40(%rax),%r13 2794 mov -48(%rax),%r14 2795 mov -56(%rax),%r15 2796 mov %rbx,144($context) # restore context->Rbx 2797 mov %rbp,160($context) # restore context->Rbp 2798 mov %r12,216($context) # restore context->R12 2799 mov %r13,224($context) # restore context->R13 2800 mov %r14,232($context) # restore context->R14 2801 mov %r15,240($context) # restore context->R15 2802 2803.Lin_cbc_prologue: 2804 mov 8(%rax),%rdi 2805 mov 16(%rax),%rsi 2806 mov %rax,152($context) # restore context->Rsp 2807 mov %rsi,168($context) # restore context->Rsi 2808 mov %rdi,176($context) # restore context->Rdi 2809 2810.Lcommon_seh_exit: 2811 2812 mov 40($disp),%rdi # disp->ContextRecord 2813 mov $context,%rsi # context 2814 mov \$`1232/8`,%ecx # sizeof(CONTEXT) 2815 .long 0xa548f3fc # cld; rep movsq 2816 2817 mov $disp,%rsi 2818 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 2819 mov 8(%rsi),%rdx # arg2, disp->ImageBase 2820 mov 0(%rsi),%r8 # arg3, disp->ControlPc 2821 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 2822 mov 40(%rsi),%r10 # disp->ContextRecord 2823 lea 56(%rsi),%r11 # &disp->HandlerData 2824 lea 24(%rsi),%r12 # &disp->EstablisherFrame 2825 mov %r10,32(%rsp) # arg5 2826 mov %r11,40(%rsp) # arg6 2827 mov %r12,48(%rsp) # arg7 2828 mov %rcx,56(%rsp) # arg8, (NULL) 2829 call *__imp_RtlVirtualUnwind(%rip) 2830 2831 mov \$1,%eax # ExceptionContinueSearch 2832 add \$64,%rsp 2833 popfq 2834 pop %r15 2835 pop %r14 2836 pop %r13 2837 pop %r12 2838 pop %rbp 2839 pop %rbx 2840 pop %rdi 2841 pop %rsi 2842 ret 2843.size cbc_se_handler,.-cbc_se_handler 2844 2845.section .pdata 2846.align 4 2847 .rva .LSEH_begin_aes_nohw_encrypt 2848 .rva .LSEH_end_aes_nohw_encrypt 2849 .rva .LSEH_info_aes_nohw_encrypt 2850 2851 .rva .LSEH_begin_aes_nohw_decrypt 2852 .rva .LSEH_end_aes_nohw_decrypt 2853 .rva .LSEH_info_aes_nohw_decrypt 2854 2855 .rva .LSEH_begin_aes_nohw_set_encrypt_key 2856 .rva .LSEH_end_aes_nohw_set_encrypt_key 2857 .rva .LSEH_info_aes_nohw_set_encrypt_key 2858 2859 .rva .LSEH_begin_aes_nohw_set_decrypt_key 2860 .rva .LSEH_end_aes_nohw_set_decrypt_key 2861 .rva .LSEH_info_aes_nohw_set_decrypt_key 2862 2863 .rva .LSEH_begin_aes_nohw_cbc_encrypt 2864 .rva .LSEH_end_aes_nohw_cbc_encrypt 2865 .rva .LSEH_info_aes_nohw_cbc_encrypt 2866 2867.section .xdata 2868.align 8 2869.LSEH_info_aes_nohw_encrypt: 2870 .byte 9,0,0,0 2871 .rva block_se_handler 2872 .rva .Lenc_prologue,.Lenc_epilogue # HandlerData[] 2873.LSEH_info_aes_nohw_decrypt: 2874 .byte 9,0,0,0 2875 .rva block_se_handler 2876 .rva .Ldec_prologue,.Ldec_epilogue # HandlerData[] 2877.LSEH_info_aes_nohw_set_encrypt_key: 2878 .byte 9,0,0,0 2879 .rva key_se_handler 2880 .rva .Lenc_key_prologue,.Lenc_key_epilogue # HandlerData[] 2881.LSEH_info_aes_nohw_set_decrypt_key: 2882 .byte 9,0,0,0 2883 .rva key_se_handler 2884 .rva .Ldec_key_prologue,.Ldec_key_epilogue # HandlerData[] 2885.LSEH_info_aes_nohw_cbc_encrypt: 2886 .byte 9,0,0,0 2887 .rva cbc_se_handler 2888___ 2889} 2890 2891$code =~ s/\`([^\`]*)\`/eval($1)/gem; 2892 2893print $code; 2894 2895close STDOUT; 2896