1238384Sjkim#!/usr/bin/env perl 2238384Sjkim# 3238384Sjkim# ==================================================================== 4238384Sjkim# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL 5238384Sjkim# project. The module is, however, dual licensed under OpenSSL and 6238384Sjkim# CRYPTOGAMS licenses depending on where you obtain it. For further 7238384Sjkim# details see http://www.openssl.org/~appro/cryptogams/. 8238384Sjkim# ==================================================================== 9238384Sjkim# 10238384Sjkim# March, June 2010 11238384Sjkim# 12238384Sjkim# The module implements "4-bit" GCM GHASH function and underlying 13238384Sjkim# single multiplication operation in GF(2^128). "4-bit" means that 14238384Sjkim# it uses 256 bytes per-key table [+128 bytes shared table]. GHASH 15238384Sjkim# function features so called "528B" variant utilizing additional 16238384Sjkim# 256+16 bytes of per-key storage [+512 bytes shared table]. 17238384Sjkim# Performance results are for this streamed GHASH subroutine and are 18238384Sjkim# expressed in cycles per processed byte, less is better: 19238384Sjkim# 20238384Sjkim# gcc 3.4.x(*) assembler 21238384Sjkim# 22238384Sjkim# P4 28.6 14.0 +100% 23238384Sjkim# Opteron 19.3 7.7 +150% 24238384Sjkim# Core2 17.8 8.1(**) +120% 25238384Sjkim# 26238384Sjkim# (*) comparison is not completely fair, because C results are 27238384Sjkim# for vanilla "256B" implementation, while assembler results 28238384Sjkim# are for "528B";-) 29238384Sjkim# (**) it's mystery [to me] why Core2 result is not same as for 30238384Sjkim# Opteron; 31238384Sjkim 32238384Sjkim# May 2010 33238384Sjkim# 34238384Sjkim# Add PCLMULQDQ version performing at 2.02 cycles per processed byte. 35238384Sjkim# See ghash-x86.pl for background information and details about coding 36238384Sjkim# techniques. 37238384Sjkim# 38238384Sjkim# Special thanks to David Woodhouse <dwmw2@infradead.org> for 39238384Sjkim# providing access to a Westmere-based system on behalf of Intel 40238384Sjkim# Open Source Technology Centre. 41238384Sjkim 42238384Sjkim$flavour = shift; 43238384Sjkim$output = shift; 44238384Sjkimif ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 45238384Sjkim 46238384Sjkim$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 47238384Sjkim 48238384Sjkim$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 49238384Sjkim( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 50238384Sjkim( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 51238384Sjkimdie "can't locate x86_64-xlate.pl"; 52238384Sjkim 53246772Sjkimopen OUT,"| \"$^X\" $xlate $flavour $output"; 54246772Sjkim*STDOUT=*OUT; 55238384Sjkim 56238384Sjkim# common register layout 57238384Sjkim$nlo="%rax"; 58238384Sjkim$nhi="%rbx"; 59238384Sjkim$Zlo="%r8"; 60238384Sjkim$Zhi="%r9"; 61238384Sjkim$tmp="%r10"; 62238384Sjkim$rem_4bit = "%r11"; 63238384Sjkim 64238384Sjkim$Xi="%rdi"; 65238384Sjkim$Htbl="%rsi"; 66238384Sjkim 67238384Sjkim# per-function register layout 68238384Sjkim$cnt="%rcx"; 69238384Sjkim$rem="%rdx"; 70238384Sjkim 71238384Sjkimsub LB() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1l/ or 72238384Sjkim $r =~ s/%[er]([sd]i)/%\1l/ or 73238384Sjkim $r =~ s/%[er](bp)/%\1l/ or 74238384Sjkim $r =~ s/%(r[0-9]+)[d]?/%\1b/; $r; } 75238384Sjkim 76238384Sjkimsub AUTOLOAD() # thunk [simplified] 32-bit style perlasm 77238384Sjkim{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; 78238384Sjkim my $arg = pop; 79238384Sjkim $arg = "\$$arg" if ($arg*1 eq $arg); 80238384Sjkim $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; 81238384Sjkim} 82238384Sjkim 83238384Sjkim{ my $N; 84238384Sjkim sub loop() { 85238384Sjkim my $inp = shift; 86238384Sjkim 87238384Sjkim $N++; 88238384Sjkim$code.=<<___; 89238384Sjkim xor $nlo,$nlo 90238384Sjkim xor $nhi,$nhi 91238384Sjkim mov `&LB("$Zlo")`,`&LB("$nlo")` 92238384Sjkim mov `&LB("$Zlo")`,`&LB("$nhi")` 93238384Sjkim shl \$4,`&LB("$nlo")` 94238384Sjkim mov \$14,$cnt 95238384Sjkim mov 8($Htbl,$nlo),$Zlo 96238384Sjkim mov ($Htbl,$nlo),$Zhi 97238384Sjkim and \$0xf0,`&LB("$nhi")` 98238384Sjkim mov $Zlo,$rem 99238384Sjkim jmp .Loop$N 100238384Sjkim 101238384Sjkim.align 16 102238384Sjkim.Loop$N: 103238384Sjkim shr \$4,$Zlo 104238384Sjkim and \$0xf,$rem 105238384Sjkim mov $Zhi,$tmp 106238384Sjkim mov ($inp,$cnt),`&LB("$nlo")` 107238384Sjkim shr \$4,$Zhi 108238384Sjkim xor 8($Htbl,$nhi),$Zlo 109238384Sjkim shl \$60,$tmp 110238384Sjkim xor ($Htbl,$nhi),$Zhi 111238384Sjkim mov `&LB("$nlo")`,`&LB("$nhi")` 112238384Sjkim xor ($rem_4bit,$rem,8),$Zhi 113238384Sjkim mov $Zlo,$rem 114238384Sjkim shl \$4,`&LB("$nlo")` 115238384Sjkim xor $tmp,$Zlo 116238384Sjkim dec $cnt 117238384Sjkim js .Lbreak$N 118238384Sjkim 119238384Sjkim shr \$4,$Zlo 120238384Sjkim and \$0xf,$rem 121238384Sjkim mov $Zhi,$tmp 122238384Sjkim shr \$4,$Zhi 123238384Sjkim xor 8($Htbl,$nlo),$Zlo 124238384Sjkim shl \$60,$tmp 125238384Sjkim xor ($Htbl,$nlo),$Zhi 126238384Sjkim and \$0xf0,`&LB("$nhi")` 127238384Sjkim xor ($rem_4bit,$rem,8),$Zhi 128238384Sjkim mov $Zlo,$rem 129238384Sjkim xor $tmp,$Zlo 130238384Sjkim jmp .Loop$N 131238384Sjkim 132238384Sjkim.align 16 133238384Sjkim.Lbreak$N: 134238384Sjkim shr \$4,$Zlo 135238384Sjkim and \$0xf,$rem 136238384Sjkim mov $Zhi,$tmp 137238384Sjkim shr \$4,$Zhi 138238384Sjkim xor 8($Htbl,$nlo),$Zlo 139238384Sjkim shl \$60,$tmp 140238384Sjkim xor ($Htbl,$nlo),$Zhi 141238384Sjkim and \$0xf0,`&LB("$nhi")` 142238384Sjkim xor ($rem_4bit,$rem,8),$Zhi 143238384Sjkim mov $Zlo,$rem 144238384Sjkim xor $tmp,$Zlo 145238384Sjkim 146238384Sjkim shr \$4,$Zlo 147238384Sjkim and \$0xf,$rem 148238384Sjkim mov $Zhi,$tmp 149238384Sjkim shr \$4,$Zhi 150238384Sjkim xor 8($Htbl,$nhi),$Zlo 151238384Sjkim shl \$60,$tmp 152238384Sjkim xor ($Htbl,$nhi),$Zhi 153238384Sjkim xor $tmp,$Zlo 154238384Sjkim xor ($rem_4bit,$rem,8),$Zhi 155238384Sjkim 156238384Sjkim bswap $Zlo 157238384Sjkim bswap $Zhi 158238384Sjkim___ 159238384Sjkim}} 160238384Sjkim 161238384Sjkim$code=<<___; 162238384Sjkim.text 163238384Sjkim 164238384Sjkim.globl gcm_gmult_4bit 165238384Sjkim.type gcm_gmult_4bit,\@function,2 166238384Sjkim.align 16 167238384Sjkimgcm_gmult_4bit: 168238384Sjkim push %rbx 169238384Sjkim push %rbp # %rbp and %r12 are pushed exclusively in 170238384Sjkim push %r12 # order to reuse Win64 exception handler... 171238384Sjkim.Lgmult_prologue: 172238384Sjkim 173238384Sjkim movzb 15($Xi),$Zlo 174238384Sjkim lea .Lrem_4bit(%rip),$rem_4bit 175238384Sjkim___ 176238384Sjkim &loop ($Xi); 177238384Sjkim$code.=<<___; 178238384Sjkim mov $Zlo,8($Xi) 179238384Sjkim mov $Zhi,($Xi) 180238384Sjkim 181238384Sjkim mov 16(%rsp),%rbx 182238384Sjkim lea 24(%rsp),%rsp 183238384Sjkim.Lgmult_epilogue: 184238384Sjkim ret 185238384Sjkim.size gcm_gmult_4bit,.-gcm_gmult_4bit 186238384Sjkim___ 187238384Sjkim 188238384Sjkim# per-function register layout 189238384Sjkim$inp="%rdx"; 190238384Sjkim$len="%rcx"; 191238384Sjkim$rem_8bit=$rem_4bit; 192238384Sjkim 193238384Sjkim$code.=<<___; 194238384Sjkim.globl gcm_ghash_4bit 195238384Sjkim.type gcm_ghash_4bit,\@function,4 196238384Sjkim.align 16 197238384Sjkimgcm_ghash_4bit: 198238384Sjkim push %rbx 199238384Sjkim push %rbp 200238384Sjkim push %r12 201238384Sjkim push %r13 202238384Sjkim push %r14 203238384Sjkim push %r15 204238384Sjkim sub \$280,%rsp 205238384Sjkim.Lghash_prologue: 206238384Sjkim mov $inp,%r14 # reassign couple of args 207238384Sjkim mov $len,%r15 208238384Sjkim___ 209238384Sjkim{ my $inp="%r14"; 210238384Sjkim my $dat="%edx"; 211238384Sjkim my $len="%r15"; 212238384Sjkim my @nhi=("%ebx","%ecx"); 213238384Sjkim my @rem=("%r12","%r13"); 214238384Sjkim my $Hshr4="%rbp"; 215238384Sjkim 216238384Sjkim &sub ($Htbl,-128); # size optimization 217238384Sjkim &lea ($Hshr4,"16+128(%rsp)"); 218238384Sjkim { my @lo =($nlo,$nhi); 219238384Sjkim my @hi =($Zlo,$Zhi); 220238384Sjkim 221238384Sjkim &xor ($dat,$dat); 222238384Sjkim for ($i=0,$j=-2;$i<18;$i++,$j++) { 223238384Sjkim &mov ("$j(%rsp)",&LB($dat)) if ($i>1); 224238384Sjkim &or ($lo[0],$tmp) if ($i>1); 225238384Sjkim &mov (&LB($dat),&LB($lo[1])) if ($i>0 && $i<17); 226238384Sjkim &shr ($lo[1],4) if ($i>0 && $i<17); 227238384Sjkim &mov ($tmp,$hi[1]) if ($i>0 && $i<17); 228238384Sjkim &shr ($hi[1],4) if ($i>0 && $i<17); 229238384Sjkim &mov ("8*$j($Hshr4)",$hi[0]) if ($i>1); 230238384Sjkim &mov ($hi[0],"16*$i+0-128($Htbl)") if ($i<16); 231238384Sjkim &shl (&LB($dat),4) if ($i>0 && $i<17); 232238384Sjkim &mov ("8*$j-128($Hshr4)",$lo[0]) if ($i>1); 233238384Sjkim &mov ($lo[0],"16*$i+8-128($Htbl)") if ($i<16); 234238384Sjkim &shl ($tmp,60) if ($i>0 && $i<17); 235238384Sjkim 236238384Sjkim push (@lo,shift(@lo)); 237238384Sjkim push (@hi,shift(@hi)); 238238384Sjkim } 239238384Sjkim } 240238384Sjkim &add ($Htbl,-128); 241238384Sjkim &mov ($Zlo,"8($Xi)"); 242238384Sjkim &mov ($Zhi,"0($Xi)"); 243238384Sjkim &add ($len,$inp); # pointer to the end of data 244238384Sjkim &lea ($rem_8bit,".Lrem_8bit(%rip)"); 245238384Sjkim &jmp (".Louter_loop"); 246238384Sjkim 247238384Sjkim$code.=".align 16\n.Louter_loop:\n"; 248238384Sjkim &xor ($Zhi,"($inp)"); 249238384Sjkim &mov ("%rdx","8($inp)"); 250238384Sjkim &lea ($inp,"16($inp)"); 251238384Sjkim &xor ("%rdx",$Zlo); 252238384Sjkim &mov ("($Xi)",$Zhi); 253238384Sjkim &mov ("8($Xi)","%rdx"); 254238384Sjkim &shr ("%rdx",32); 255238384Sjkim 256238384Sjkim &xor ($nlo,$nlo); 257238384Sjkim &rol ($dat,8); 258238384Sjkim &mov (&LB($nlo),&LB($dat)); 259238384Sjkim &movz ($nhi[0],&LB($dat)); 260238384Sjkim &shl (&LB($nlo),4); 261238384Sjkim &shr ($nhi[0],4); 262238384Sjkim 263238384Sjkim for ($j=11,$i=0;$i<15;$i++) { 264238384Sjkim &rol ($dat,8); 265238384Sjkim &xor ($Zlo,"8($Htbl,$nlo)") if ($i>0); 266238384Sjkim &xor ($Zhi,"($Htbl,$nlo)") if ($i>0); 267238384Sjkim &mov ($Zlo,"8($Htbl,$nlo)") if ($i==0); 268238384Sjkim &mov ($Zhi,"($Htbl,$nlo)") if ($i==0); 269238384Sjkim 270238384Sjkim &mov (&LB($nlo),&LB($dat)); 271238384Sjkim &xor ($Zlo,$tmp) if ($i>0); 272238384Sjkim &movzw ($rem[1],"($rem_8bit,$rem[1],2)") if ($i>0); 273238384Sjkim 274238384Sjkim &movz ($nhi[1],&LB($dat)); 275238384Sjkim &shl (&LB($nlo),4); 276238384Sjkim &movzb ($rem[0],"(%rsp,$nhi[0])"); 277238384Sjkim 278238384Sjkim &shr ($nhi[1],4) if ($i<14); 279238384Sjkim &and ($nhi[1],0xf0) if ($i==14); 280238384Sjkim &shl ($rem[1],48) if ($i>0); 281238384Sjkim &xor ($rem[0],$Zlo); 282238384Sjkim 283238384Sjkim &mov ($tmp,$Zhi); 284238384Sjkim &xor ($Zhi,$rem[1]) if ($i>0); 285238384Sjkim &shr ($Zlo,8); 286238384Sjkim 287238384Sjkim &movz ($rem[0],&LB($rem[0])); 288238384Sjkim &mov ($dat,"$j($Xi)") if (--$j%4==0); 289238384Sjkim &shr ($Zhi,8); 290238384Sjkim 291238384Sjkim &xor ($Zlo,"-128($Hshr4,$nhi[0],8)"); 292238384Sjkim &shl ($tmp,56); 293238384Sjkim &xor ($Zhi,"($Hshr4,$nhi[0],8)"); 294238384Sjkim 295238384Sjkim unshift (@nhi,pop(@nhi)); # "rotate" registers 296238384Sjkim unshift (@rem,pop(@rem)); 297238384Sjkim } 298238384Sjkim &movzw ($rem[1],"($rem_8bit,$rem[1],2)"); 299238384Sjkim &xor ($Zlo,"8($Htbl,$nlo)"); 300238384Sjkim &xor ($Zhi,"($Htbl,$nlo)"); 301238384Sjkim 302238384Sjkim &shl ($rem[1],48); 303238384Sjkim &xor ($Zlo,$tmp); 304238384Sjkim 305238384Sjkim &xor ($Zhi,$rem[1]); 306238384Sjkim &movz ($rem[0],&LB($Zlo)); 307238384Sjkim &shr ($Zlo,4); 308238384Sjkim 309238384Sjkim &mov ($tmp,$Zhi); 310238384Sjkim &shl (&LB($rem[0]),4); 311238384Sjkim &shr ($Zhi,4); 312238384Sjkim 313238384Sjkim &xor ($Zlo,"8($Htbl,$nhi[0])"); 314238384Sjkim &movzw ($rem[0],"($rem_8bit,$rem[0],2)"); 315238384Sjkim &shl ($tmp,60); 316238384Sjkim 317238384Sjkim &xor ($Zhi,"($Htbl,$nhi[0])"); 318238384Sjkim &xor ($Zlo,$tmp); 319238384Sjkim &shl ($rem[0],48); 320238384Sjkim 321238384Sjkim &bswap ($Zlo); 322238384Sjkim &xor ($Zhi,$rem[0]); 323238384Sjkim 324238384Sjkim &bswap ($Zhi); 325238384Sjkim &cmp ($inp,$len); 326238384Sjkim &jb (".Louter_loop"); 327238384Sjkim} 328238384Sjkim$code.=<<___; 329238384Sjkim mov $Zlo,8($Xi) 330238384Sjkim mov $Zhi,($Xi) 331238384Sjkim 332238384Sjkim lea 280(%rsp),%rsi 333238384Sjkim mov 0(%rsi),%r15 334238384Sjkim mov 8(%rsi),%r14 335238384Sjkim mov 16(%rsi),%r13 336238384Sjkim mov 24(%rsi),%r12 337238384Sjkim mov 32(%rsi),%rbp 338238384Sjkim mov 40(%rsi),%rbx 339238384Sjkim lea 48(%rsi),%rsp 340238384Sjkim.Lghash_epilogue: 341238384Sjkim ret 342238384Sjkim.size gcm_ghash_4bit,.-gcm_ghash_4bit 343238384Sjkim___ 344238384Sjkim 345238384Sjkim###################################################################### 346238384Sjkim# PCLMULQDQ version. 347238384Sjkim 348238384Sjkim@_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order 349238384Sjkim ("%rdi","%rsi","%rdx","%rcx"); # Unix order 350238384Sjkim 351238384Sjkim($Xi,$Xhi)=("%xmm0","%xmm1"); $Hkey="%xmm2"; 352238384Sjkim($T1,$T2,$T3)=("%xmm3","%xmm4","%xmm5"); 353238384Sjkim 354238384Sjkimsub clmul64x64_T2 { # minimal register pressure 355238384Sjkimmy ($Xhi,$Xi,$Hkey,$modulo)=@_; 356238384Sjkim 357238384Sjkim$code.=<<___ if (!defined($modulo)); 358238384Sjkim movdqa $Xi,$Xhi # 359238384Sjkim pshufd \$0b01001110,$Xi,$T1 360238384Sjkim pshufd \$0b01001110,$Hkey,$T2 361238384Sjkim pxor $Xi,$T1 # 362238384Sjkim pxor $Hkey,$T2 363238384Sjkim___ 364238384Sjkim$code.=<<___; 365238384Sjkim pclmulqdq \$0x00,$Hkey,$Xi ####### 366238384Sjkim pclmulqdq \$0x11,$Hkey,$Xhi ####### 367238384Sjkim pclmulqdq \$0x00,$T2,$T1 ####### 368238384Sjkim pxor $Xi,$T1 # 369238384Sjkim pxor $Xhi,$T1 # 370238384Sjkim 371238384Sjkim movdqa $T1,$T2 # 372238384Sjkim psrldq \$8,$T1 373238384Sjkim pslldq \$8,$T2 # 374238384Sjkim pxor $T1,$Xhi 375238384Sjkim pxor $T2,$Xi # 376238384Sjkim___ 377238384Sjkim} 378238384Sjkim 379238384Sjkimsub reduction_alg9 { # 17/13 times faster than Intel version 380238384Sjkimmy ($Xhi,$Xi) = @_; 381238384Sjkim 382238384Sjkim$code.=<<___; 383238384Sjkim # 1st phase 384238384Sjkim movdqa $Xi,$T1 # 385238384Sjkim psllq \$1,$Xi 386238384Sjkim pxor $T1,$Xi # 387238384Sjkim psllq \$5,$Xi # 388238384Sjkim pxor $T1,$Xi # 389238384Sjkim psllq \$57,$Xi # 390238384Sjkim movdqa $Xi,$T2 # 391238384Sjkim pslldq \$8,$Xi 392238384Sjkim psrldq \$8,$T2 # 393238384Sjkim pxor $T1,$Xi 394238384Sjkim pxor $T2,$Xhi # 395238384Sjkim 396238384Sjkim # 2nd phase 397238384Sjkim movdqa $Xi,$T2 398238384Sjkim psrlq \$5,$Xi 399238384Sjkim pxor $T2,$Xi # 400238384Sjkim psrlq \$1,$Xi # 401238384Sjkim pxor $T2,$Xi # 402238384Sjkim pxor $Xhi,$T2 403238384Sjkim psrlq \$1,$Xi # 404238384Sjkim pxor $T2,$Xi # 405238384Sjkim___ 406238384Sjkim} 407238384Sjkim 408238384Sjkim{ my ($Htbl,$Xip)=@_4args; 409238384Sjkim 410238384Sjkim$code.=<<___; 411238384Sjkim.globl gcm_init_clmul 412238384Sjkim.type gcm_init_clmul,\@abi-omnipotent 413238384Sjkim.align 16 414238384Sjkimgcm_init_clmul: 415238384Sjkim movdqu ($Xip),$Hkey 416238384Sjkim pshufd \$0b01001110,$Hkey,$Hkey # dword swap 417238384Sjkim 418238384Sjkim # <<1 twist 419238384Sjkim pshufd \$0b11111111,$Hkey,$T2 # broadcast uppermost dword 420238384Sjkim movdqa $Hkey,$T1 421238384Sjkim psllq \$1,$Hkey 422238384Sjkim pxor $T3,$T3 # 423238384Sjkim psrlq \$63,$T1 424238384Sjkim pcmpgtd $T2,$T3 # broadcast carry bit 425238384Sjkim pslldq \$8,$T1 426238384Sjkim por $T1,$Hkey # H<<=1 427238384Sjkim 428238384Sjkim # magic reduction 429238384Sjkim pand .L0x1c2_polynomial(%rip),$T3 430238384Sjkim pxor $T3,$Hkey # if(carry) H^=0x1c2_polynomial 431238384Sjkim 432238384Sjkim # calculate H^2 433238384Sjkim movdqa $Hkey,$Xi 434238384Sjkim___ 435238384Sjkim &clmul64x64_T2 ($Xhi,$Xi,$Hkey); 436238384Sjkim &reduction_alg9 ($Xhi,$Xi); 437238384Sjkim$code.=<<___; 438238384Sjkim movdqu $Hkey,($Htbl) # save H 439238384Sjkim movdqu $Xi,16($Htbl) # save H^2 440238384Sjkim ret 441238384Sjkim.size gcm_init_clmul,.-gcm_init_clmul 442238384Sjkim___ 443238384Sjkim} 444238384Sjkim 445238384Sjkim{ my ($Xip,$Htbl)=@_4args; 446238384Sjkim 447238384Sjkim$code.=<<___; 448238384Sjkim.globl gcm_gmult_clmul 449238384Sjkim.type gcm_gmult_clmul,\@abi-omnipotent 450238384Sjkim.align 16 451238384Sjkimgcm_gmult_clmul: 452238384Sjkim movdqu ($Xip),$Xi 453238384Sjkim movdqa .Lbswap_mask(%rip),$T3 454238384Sjkim movdqu ($Htbl),$Hkey 455238384Sjkim pshufb $T3,$Xi 456238384Sjkim___ 457238384Sjkim &clmul64x64_T2 ($Xhi,$Xi,$Hkey); 458238384Sjkim &reduction_alg9 ($Xhi,$Xi); 459238384Sjkim$code.=<<___; 460238384Sjkim pshufb $T3,$Xi 461238384Sjkim movdqu $Xi,($Xip) 462238384Sjkim ret 463238384Sjkim.size gcm_gmult_clmul,.-gcm_gmult_clmul 464238384Sjkim___ 465238384Sjkim} 466238384Sjkim 467238384Sjkim{ my ($Xip,$Htbl,$inp,$len)=@_4args; 468238384Sjkim my $Xn="%xmm6"; 469238384Sjkim my $Xhn="%xmm7"; 470238384Sjkim my $Hkey2="%xmm8"; 471238384Sjkim my $T1n="%xmm9"; 472238384Sjkim my $T2n="%xmm10"; 473238384Sjkim 474238384Sjkim$code.=<<___; 475238384Sjkim.globl gcm_ghash_clmul 476238384Sjkim.type gcm_ghash_clmul,\@abi-omnipotent 477238384Sjkim.align 16 478238384Sjkimgcm_ghash_clmul: 479238384Sjkim___ 480238384Sjkim$code.=<<___ if ($win64); 481238384Sjkim.LSEH_begin_gcm_ghash_clmul: 482238384Sjkim # I can't trust assembler to use specific encoding:-( 483238384Sjkim .byte 0x48,0x83,0xec,0x58 #sub \$0x58,%rsp 484238384Sjkim .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) 485238384Sjkim .byte 0x0f,0x29,0x7c,0x24,0x10 #movdqa %xmm7,0x10(%rsp) 486238384Sjkim .byte 0x44,0x0f,0x29,0x44,0x24,0x20 #movaps %xmm8,0x20(%rsp) 487238384Sjkim .byte 0x44,0x0f,0x29,0x4c,0x24,0x30 #movaps %xmm9,0x30(%rsp) 488238384Sjkim .byte 0x44,0x0f,0x29,0x54,0x24,0x40 #movaps %xmm10,0x40(%rsp) 489238384Sjkim___ 490238384Sjkim$code.=<<___; 491238384Sjkim movdqa .Lbswap_mask(%rip),$T3 492238384Sjkim 493238384Sjkim movdqu ($Xip),$Xi 494238384Sjkim movdqu ($Htbl),$Hkey 495238384Sjkim pshufb $T3,$Xi 496238384Sjkim 497238384Sjkim sub \$0x10,$len 498238384Sjkim jz .Lodd_tail 499238384Sjkim 500238384Sjkim movdqu 16($Htbl),$Hkey2 501238384Sjkim ####### 502238384Sjkim # Xi+2 =[H*(Ii+1 + Xi+1)] mod P = 503238384Sjkim # [(H*Ii+1) + (H*Xi+1)] mod P = 504238384Sjkim # [(H*Ii+1) + H^2*(Ii+Xi)] mod P 505238384Sjkim # 506238384Sjkim movdqu ($inp),$T1 # Ii 507238384Sjkim movdqu 16($inp),$Xn # Ii+1 508238384Sjkim pshufb $T3,$T1 509238384Sjkim pshufb $T3,$Xn 510238384Sjkim pxor $T1,$Xi # Ii+Xi 511238384Sjkim___ 512238384Sjkim &clmul64x64_T2 ($Xhn,$Xn,$Hkey); # H*Ii+1 513238384Sjkim$code.=<<___; 514238384Sjkim movdqa $Xi,$Xhi # 515238384Sjkim pshufd \$0b01001110,$Xi,$T1 516238384Sjkim pshufd \$0b01001110,$Hkey2,$T2 517238384Sjkim pxor $Xi,$T1 # 518238384Sjkim pxor $Hkey2,$T2 519238384Sjkim 520238384Sjkim lea 32($inp),$inp # i+=2 521238384Sjkim sub \$0x20,$len 522238384Sjkim jbe .Leven_tail 523238384Sjkim 524238384Sjkim.Lmod_loop: 525238384Sjkim___ 526238384Sjkim &clmul64x64_T2 ($Xhi,$Xi,$Hkey2,1); # H^2*(Ii+Xi) 527238384Sjkim$code.=<<___; 528238384Sjkim movdqu ($inp),$T1 # Ii 529238384Sjkim pxor $Xn,$Xi # (H*Ii+1) + H^2*(Ii+Xi) 530238384Sjkim pxor $Xhn,$Xhi 531238384Sjkim 532238384Sjkim movdqu 16($inp),$Xn # Ii+1 533238384Sjkim pshufb $T3,$T1 534238384Sjkim pshufb $T3,$Xn 535238384Sjkim 536238384Sjkim movdqa $Xn,$Xhn # 537238384Sjkim pshufd \$0b01001110,$Xn,$T1n 538238384Sjkim pshufd \$0b01001110,$Hkey,$T2n 539238384Sjkim pxor $Xn,$T1n # 540238384Sjkim pxor $Hkey,$T2n 541238384Sjkim pxor $T1,$Xhi # "Ii+Xi", consume early 542238384Sjkim 543238384Sjkim movdqa $Xi,$T1 # 1st phase 544238384Sjkim psllq \$1,$Xi 545238384Sjkim pxor $T1,$Xi # 546238384Sjkim psllq \$5,$Xi # 547238384Sjkim pxor $T1,$Xi # 548238384Sjkim pclmulqdq \$0x00,$Hkey,$Xn ####### 549238384Sjkim psllq \$57,$Xi # 550238384Sjkim movdqa $Xi,$T2 # 551238384Sjkim pslldq \$8,$Xi 552238384Sjkim psrldq \$8,$T2 # 553238384Sjkim pxor $T1,$Xi 554238384Sjkim pxor $T2,$Xhi # 555238384Sjkim 556238384Sjkim pclmulqdq \$0x11,$Hkey,$Xhn ####### 557238384Sjkim movdqa $Xi,$T2 # 2nd phase 558238384Sjkim psrlq \$5,$Xi 559238384Sjkim pxor $T2,$Xi # 560238384Sjkim psrlq \$1,$Xi # 561238384Sjkim pxor $T2,$Xi # 562238384Sjkim pxor $Xhi,$T2 563238384Sjkim psrlq \$1,$Xi # 564238384Sjkim pxor $T2,$Xi # 565238384Sjkim 566238384Sjkim pclmulqdq \$0x00,$T2n,$T1n ####### 567238384Sjkim movdqa $Xi,$Xhi # 568238384Sjkim pshufd \$0b01001110,$Xi,$T1 569238384Sjkim pshufd \$0b01001110,$Hkey2,$T2 570238384Sjkim pxor $Xi,$T1 # 571238384Sjkim pxor $Hkey2,$T2 572238384Sjkim 573238384Sjkim pxor $Xn,$T1n # 574238384Sjkim pxor $Xhn,$T1n # 575238384Sjkim movdqa $T1n,$T2n # 576238384Sjkim psrldq \$8,$T1n 577238384Sjkim pslldq \$8,$T2n # 578238384Sjkim pxor $T1n,$Xhn 579238384Sjkim pxor $T2n,$Xn # 580238384Sjkim 581238384Sjkim lea 32($inp),$inp 582238384Sjkim sub \$0x20,$len 583238384Sjkim ja .Lmod_loop 584238384Sjkim 585238384Sjkim.Leven_tail: 586238384Sjkim___ 587238384Sjkim &clmul64x64_T2 ($Xhi,$Xi,$Hkey2,1); # H^2*(Ii+Xi) 588238384Sjkim$code.=<<___; 589238384Sjkim pxor $Xn,$Xi # (H*Ii+1) + H^2*(Ii+Xi) 590238384Sjkim pxor $Xhn,$Xhi 591238384Sjkim___ 592238384Sjkim &reduction_alg9 ($Xhi,$Xi); 593238384Sjkim$code.=<<___; 594238384Sjkim test $len,$len 595238384Sjkim jnz .Ldone 596238384Sjkim 597238384Sjkim.Lodd_tail: 598238384Sjkim movdqu ($inp),$T1 # Ii 599238384Sjkim pshufb $T3,$T1 600238384Sjkim pxor $T1,$Xi # Ii+Xi 601238384Sjkim___ 602238384Sjkim &clmul64x64_T2 ($Xhi,$Xi,$Hkey); # H*(Ii+Xi) 603238384Sjkim &reduction_alg9 ($Xhi,$Xi); 604238384Sjkim$code.=<<___; 605238384Sjkim.Ldone: 606238384Sjkim pshufb $T3,$Xi 607238384Sjkim movdqu $Xi,($Xip) 608238384Sjkim___ 609238384Sjkim$code.=<<___ if ($win64); 610238384Sjkim movaps (%rsp),%xmm6 611238384Sjkim movaps 0x10(%rsp),%xmm7 612238384Sjkim movaps 0x20(%rsp),%xmm8 613238384Sjkim movaps 0x30(%rsp),%xmm9 614238384Sjkim movaps 0x40(%rsp),%xmm10 615238384Sjkim add \$0x58,%rsp 616238384Sjkim___ 617238384Sjkim$code.=<<___; 618238384Sjkim ret 619238384Sjkim.LSEH_end_gcm_ghash_clmul: 620238384Sjkim.size gcm_ghash_clmul,.-gcm_ghash_clmul 621238384Sjkim___ 622238384Sjkim} 623238384Sjkim 624238384Sjkim$code.=<<___; 625238384Sjkim.align 64 626238384Sjkim.Lbswap_mask: 627238384Sjkim .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 628238384Sjkim.L0x1c2_polynomial: 629238384Sjkim .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 630238384Sjkim.align 64 631238384Sjkim.type .Lrem_4bit,\@object 632238384Sjkim.Lrem_4bit: 633238384Sjkim .long 0,`0x0000<<16`,0,`0x1C20<<16`,0,`0x3840<<16`,0,`0x2460<<16` 634238384Sjkim .long 0,`0x7080<<16`,0,`0x6CA0<<16`,0,`0x48C0<<16`,0,`0x54E0<<16` 635238384Sjkim .long 0,`0xE100<<16`,0,`0xFD20<<16`,0,`0xD940<<16`,0,`0xC560<<16` 636238384Sjkim .long 0,`0x9180<<16`,0,`0x8DA0<<16`,0,`0xA9C0<<16`,0,`0xB5E0<<16` 637238384Sjkim.type .Lrem_8bit,\@object 638238384Sjkim.Lrem_8bit: 639238384Sjkim .value 0x0000,0x01C2,0x0384,0x0246,0x0708,0x06CA,0x048C,0x054E 640238384Sjkim .value 0x0E10,0x0FD2,0x0D94,0x0C56,0x0918,0x08DA,0x0A9C,0x0B5E 641238384Sjkim .value 0x1C20,0x1DE2,0x1FA4,0x1E66,0x1B28,0x1AEA,0x18AC,0x196E 642238384Sjkim .value 0x1230,0x13F2,0x11B4,0x1076,0x1538,0x14FA,0x16BC,0x177E 643238384Sjkim .value 0x3840,0x3982,0x3BC4,0x3A06,0x3F48,0x3E8A,0x3CCC,0x3D0E 644238384Sjkim .value 0x3650,0x3792,0x35D4,0x3416,0x3158,0x309A,0x32DC,0x331E 645238384Sjkim .value 0x2460,0x25A2,0x27E4,0x2626,0x2368,0x22AA,0x20EC,0x212E 646238384Sjkim .value 0x2A70,0x2BB2,0x29F4,0x2836,0x2D78,0x2CBA,0x2EFC,0x2F3E 647238384Sjkim .value 0x7080,0x7142,0x7304,0x72C6,0x7788,0x764A,0x740C,0x75CE 648238384Sjkim .value 0x7E90,0x7F52,0x7D14,0x7CD6,0x7998,0x785A,0x7A1C,0x7BDE 649238384Sjkim .value 0x6CA0,0x6D62,0x6F24,0x6EE6,0x6BA8,0x6A6A,0x682C,0x69EE 650238384Sjkim .value 0x62B0,0x6372,0x6134,0x60F6,0x65B8,0x647A,0x663C,0x67FE 651238384Sjkim .value 0x48C0,0x4902,0x4B44,0x4A86,0x4FC8,0x4E0A,0x4C4C,0x4D8E 652238384Sjkim .value 0x46D0,0x4712,0x4554,0x4496,0x41D8,0x401A,0x425C,0x439E 653238384Sjkim .value 0x54E0,0x5522,0x5764,0x56A6,0x53E8,0x522A,0x506C,0x51AE 654238384Sjkim .value 0x5AF0,0x5B32,0x5974,0x58B6,0x5DF8,0x5C3A,0x5E7C,0x5FBE 655238384Sjkim .value 0xE100,0xE0C2,0xE284,0xE346,0xE608,0xE7CA,0xE58C,0xE44E 656238384Sjkim .value 0xEF10,0xEED2,0xEC94,0xED56,0xE818,0xE9DA,0xEB9C,0xEA5E 657238384Sjkim .value 0xFD20,0xFCE2,0xFEA4,0xFF66,0xFA28,0xFBEA,0xF9AC,0xF86E 658238384Sjkim .value 0xF330,0xF2F2,0xF0B4,0xF176,0xF438,0xF5FA,0xF7BC,0xF67E 659238384Sjkim .value 0xD940,0xD882,0xDAC4,0xDB06,0xDE48,0xDF8A,0xDDCC,0xDC0E 660238384Sjkim .value 0xD750,0xD692,0xD4D4,0xD516,0xD058,0xD19A,0xD3DC,0xD21E 661238384Sjkim .value 0xC560,0xC4A2,0xC6E4,0xC726,0xC268,0xC3AA,0xC1EC,0xC02E 662238384Sjkim .value 0xCB70,0xCAB2,0xC8F4,0xC936,0xCC78,0xCDBA,0xCFFC,0xCE3E 663238384Sjkim .value 0x9180,0x9042,0x9204,0x93C6,0x9688,0x974A,0x950C,0x94CE 664238384Sjkim .value 0x9F90,0x9E52,0x9C14,0x9DD6,0x9898,0x995A,0x9B1C,0x9ADE 665238384Sjkim .value 0x8DA0,0x8C62,0x8E24,0x8FE6,0x8AA8,0x8B6A,0x892C,0x88EE 666238384Sjkim .value 0x83B0,0x8272,0x8034,0x81F6,0x84B8,0x857A,0x873C,0x86FE 667238384Sjkim .value 0xA9C0,0xA802,0xAA44,0xAB86,0xAEC8,0xAF0A,0xAD4C,0xAC8E 668238384Sjkim .value 0xA7D0,0xA612,0xA454,0xA596,0xA0D8,0xA11A,0xA35C,0xA29E 669238384Sjkim .value 0xB5E0,0xB422,0xB664,0xB7A6,0xB2E8,0xB32A,0xB16C,0xB0AE 670238384Sjkim .value 0xBBF0,0xBA32,0xB874,0xB9B6,0xBCF8,0xBD3A,0xBF7C,0xBEBE 671238384Sjkim 672238384Sjkim.asciz "GHASH for x86_64, CRYPTOGAMS by <appro\@openssl.org>" 673238384Sjkim.align 64 674238384Sjkim___ 675238384Sjkim 676238384Sjkim# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 677238384Sjkim# CONTEXT *context,DISPATCHER_CONTEXT *disp) 678238384Sjkimif ($win64) { 679238384Sjkim$rec="%rcx"; 680238384Sjkim$frame="%rdx"; 681238384Sjkim$context="%r8"; 682238384Sjkim$disp="%r9"; 683238384Sjkim 684238384Sjkim$code.=<<___; 685238384Sjkim.extern __imp_RtlVirtualUnwind 686238384Sjkim.type se_handler,\@abi-omnipotent 687238384Sjkim.align 16 688238384Sjkimse_handler: 689238384Sjkim push %rsi 690238384Sjkim push %rdi 691238384Sjkim push %rbx 692238384Sjkim push %rbp 693238384Sjkim push %r12 694238384Sjkim push %r13 695238384Sjkim push %r14 696238384Sjkim push %r15 697238384Sjkim pushfq 698238384Sjkim sub \$64,%rsp 699238384Sjkim 700238384Sjkim mov 120($context),%rax # pull context->Rax 701238384Sjkim mov 248($context),%rbx # pull context->Rip 702238384Sjkim 703238384Sjkim mov 8($disp),%rsi # disp->ImageBase 704238384Sjkim mov 56($disp),%r11 # disp->HandlerData 705238384Sjkim 706238384Sjkim mov 0(%r11),%r10d # HandlerData[0] 707238384Sjkim lea (%rsi,%r10),%r10 # prologue label 708238384Sjkim cmp %r10,%rbx # context->Rip<prologue label 709238384Sjkim jb .Lin_prologue 710238384Sjkim 711238384Sjkim mov 152($context),%rax # pull context->Rsp 712238384Sjkim 713238384Sjkim mov 4(%r11),%r10d # HandlerData[1] 714238384Sjkim lea (%rsi,%r10),%r10 # epilogue label 715238384Sjkim cmp %r10,%rbx # context->Rip>=epilogue label 716238384Sjkim jae .Lin_prologue 717238384Sjkim 718238384Sjkim lea 24(%rax),%rax # adjust "rsp" 719238384Sjkim 720238384Sjkim mov -8(%rax),%rbx 721238384Sjkim mov -16(%rax),%rbp 722238384Sjkim mov -24(%rax),%r12 723238384Sjkim mov %rbx,144($context) # restore context->Rbx 724238384Sjkim mov %rbp,160($context) # restore context->Rbp 725238384Sjkim mov %r12,216($context) # restore context->R12 726238384Sjkim 727238384Sjkim.Lin_prologue: 728238384Sjkim mov 8(%rax),%rdi 729238384Sjkim mov 16(%rax),%rsi 730238384Sjkim mov %rax,152($context) # restore context->Rsp 731238384Sjkim mov %rsi,168($context) # restore context->Rsi 732238384Sjkim mov %rdi,176($context) # restore context->Rdi 733238384Sjkim 734238384Sjkim mov 40($disp),%rdi # disp->ContextRecord 735238384Sjkim mov $context,%rsi # context 736238384Sjkim mov \$`1232/8`,%ecx # sizeof(CONTEXT) 737238384Sjkim .long 0xa548f3fc # cld; rep movsq 738238384Sjkim 739238384Sjkim mov $disp,%rsi 740238384Sjkim xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 741238384Sjkim mov 8(%rsi),%rdx # arg2, disp->ImageBase 742238384Sjkim mov 0(%rsi),%r8 # arg3, disp->ControlPc 743238384Sjkim mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 744238384Sjkim mov 40(%rsi),%r10 # disp->ContextRecord 745238384Sjkim lea 56(%rsi),%r11 # &disp->HandlerData 746238384Sjkim lea 24(%rsi),%r12 # &disp->EstablisherFrame 747238384Sjkim mov %r10,32(%rsp) # arg5 748238384Sjkim mov %r11,40(%rsp) # arg6 749238384Sjkim mov %r12,48(%rsp) # arg7 750238384Sjkim mov %rcx,56(%rsp) # arg8, (NULL) 751238384Sjkim call *__imp_RtlVirtualUnwind(%rip) 752238384Sjkim 753238384Sjkim mov \$1,%eax # ExceptionContinueSearch 754238384Sjkim add \$64,%rsp 755238384Sjkim popfq 756238384Sjkim pop %r15 757238384Sjkim pop %r14 758238384Sjkim pop %r13 759238384Sjkim pop %r12 760238384Sjkim pop %rbp 761238384Sjkim pop %rbx 762238384Sjkim pop %rdi 763238384Sjkim pop %rsi 764238384Sjkim ret 765238384Sjkim.size se_handler,.-se_handler 766238384Sjkim 767238384Sjkim.section .pdata 768238384Sjkim.align 4 769238384Sjkim .rva .LSEH_begin_gcm_gmult_4bit 770238384Sjkim .rva .LSEH_end_gcm_gmult_4bit 771238384Sjkim .rva .LSEH_info_gcm_gmult_4bit 772238384Sjkim 773238384Sjkim .rva .LSEH_begin_gcm_ghash_4bit 774238384Sjkim .rva .LSEH_end_gcm_ghash_4bit 775238384Sjkim .rva .LSEH_info_gcm_ghash_4bit 776238384Sjkim 777238384Sjkim .rva .LSEH_begin_gcm_ghash_clmul 778238384Sjkim .rva .LSEH_end_gcm_ghash_clmul 779238384Sjkim .rva .LSEH_info_gcm_ghash_clmul 780238384Sjkim 781238384Sjkim.section .xdata 782238384Sjkim.align 8 783238384Sjkim.LSEH_info_gcm_gmult_4bit: 784238384Sjkim .byte 9,0,0,0 785238384Sjkim .rva se_handler 786238384Sjkim .rva .Lgmult_prologue,.Lgmult_epilogue # HandlerData 787238384Sjkim.LSEH_info_gcm_ghash_4bit: 788238384Sjkim .byte 9,0,0,0 789238384Sjkim .rva se_handler 790238384Sjkim .rva .Lghash_prologue,.Lghash_epilogue # HandlerData 791238384Sjkim.LSEH_info_gcm_ghash_clmul: 792238384Sjkim .byte 0x01,0x1f,0x0b,0x00 793238384Sjkim .byte 0x1f,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 794238384Sjkim .byte 0x19,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 795238384Sjkim .byte 0x13,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 796238384Sjkim .byte 0x0d,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 797238384Sjkim .byte 0x08,0x68,0x00,0x00 #movaps (rsp),xmm6 798238384Sjkim .byte 0x04,0xa2,0x00,0x00 #sub rsp,0x58 799238384Sjkim___ 800238384Sjkim} 801238384Sjkim 802238384Sjkim$code =~ s/\`([^\`]*)\`/eval($1)/gem; 803238384Sjkim 804238384Sjkimprint $code; 805238384Sjkim 806238384Sjkimclose STDOUT; 807