1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5# project. Rights for redistribution and usage in source and binary
6# forms are granted according to the OpenSSL license.
7# ====================================================================
8#
9# sha256/512_block procedure for x86_64.
10#
11# 40% improvement over compiler-generated code on Opteron. On EM64T
12# sha256 was observed to run >80% faster and sha512 - >40%. No magical
13# tricks, just straight implementation... I really wonder why gcc
14# [being armed with inline assembler] fails to generate as fast code.
15# The only thing which is cool about this module is that it's very
16# same instruction sequence used for both SHA-256 and SHA-512. In
17# former case the instructions operate on 32-bit operands, while in
18# latter - on 64-bit ones. All I had to do is to get one flavor right,
19# the other one passed the test right away:-)
20#
21# sha256_block runs in ~1005 cycles on Opteron, which gives you
22# asymptotic performance of 64*1000/1005=63.7MBps times CPU clock
23# frequency in GHz. sha512_block runs in ~1275 cycles, which results
24# in 128*1000/1275=100MBps per GHz. Is there room for improvement?
25# Well, if you compare it to IA-64 implementation, which maintains
26# X[16] in register bank[!], tends to 4 instructions per CPU clock
27# cycle and runs in 1003 cycles, 1275 is very good result for 3-way
28# issue Opteron pipeline and X[16] maintained in memory. So that *if*
29# there is a way to improve it, *then* the only way would be to try to
30# offload X[16] updates to SSE unit, but that would require "deeper"
31# loop unroll, which in turn would naturally cause size blow-up, not
32# to mention increased complexity! And once again, only *if* it's
33# actually possible to noticeably improve overall ILP, instruction
34# level parallelism, on a given CPU implementation in this case.
35#
36# Special note on Intel EM64T. While Opteron CPU exhibits perfect
37# perfromance ratio of 1.5 between 64- and 32-bit flavors [see above],
38# [currently available] EM64T CPUs apparently are far from it. On the
39# contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit
40# sha256_block:-( This is presumably because 64-bit shifts/rotates
41# apparently are not atomic instructions, but implemented in microcode.
42
43$flavour = shift;
44$output  = shift;
45if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
46
47$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
48
49$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
50( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
51( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
52die "can't locate x86_64-xlate.pl";
53
54open OUT,"| \"$^X\" $xlate $flavour $output";
55*STDOUT=*OUT;
56
57if ($output =~ /512/) {
58	$func="sha512_block_data_order";
59	$TABLE="K512";
60	$SZ=8;
61	@ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%rax","%rbx","%rcx","%rdx",
62					"%r8", "%r9", "%r10","%r11");
63	($T1,$a0,$a1,$a2)=("%r12","%r13","%r14","%r15");
64	@Sigma0=(28,34,39);
65	@Sigma1=(14,18,41);
66	@sigma0=(1,  8, 7);
67	@sigma1=(19,61, 6);
68	$rounds=80;
69} else {
70	$func="sha256_block_data_order";
71	$TABLE="K256";
72	$SZ=4;
73	@ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%eax","%ebx","%ecx","%edx",
74					"%r8d","%r9d","%r10d","%r11d");
75	($T1,$a0,$a1,$a2)=("%r12d","%r13d","%r14d","%r15d");
76	@Sigma0=( 2,13,22);
77	@Sigma1=( 6,11,25);
78	@sigma0=( 7,18, 3);
79	@sigma1=(17,19,10);
80	$rounds=64;
81}
82
83$ctx="%rdi";	# 1st arg
84$round="%rdi";	# zaps $ctx
85$inp="%rsi";	# 2nd arg
86$Tbl="%rbp";
87
88$_ctx="16*$SZ+0*8(%rsp)";
89$_inp="16*$SZ+1*8(%rsp)";
90$_end="16*$SZ+2*8(%rsp)";
91$_rsp="16*$SZ+3*8(%rsp)";
92$framesz="16*$SZ+4*8";
93
94
95sub ROUND_00_15()
96{ my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
97
98$code.=<<___;
99	mov	$e,$a0
100	mov	$e,$a1
101	mov	$f,$a2
102
103	ror	\$$Sigma1[0],$a0
104	ror	\$$Sigma1[1],$a1
105	xor	$g,$a2			# f^g
106
107	xor	$a1,$a0
108	ror	\$`$Sigma1[2]-$Sigma1[1]`,$a1
109	and	$e,$a2			# (f^g)&e
110	mov	$T1,`$SZ*($i&0xf)`(%rsp)
111
112	xor	$a1,$a0			# Sigma1(e)
113	xor	$g,$a2			# Ch(e,f,g)=((f^g)&e)^g
114	add	$h,$T1			# T1+=h
115
116	mov	$a,$h
117	add	$a0,$T1			# T1+=Sigma1(e)
118
119	add	$a2,$T1			# T1+=Ch(e,f,g)
120	mov	$a,$a0
121	mov	$a,$a1
122
123	ror	\$$Sigma0[0],$h
124	ror	\$$Sigma0[1],$a0
125	mov	$a,$a2
126	add	($Tbl,$round,$SZ),$T1	# T1+=K[round]
127
128	xor	$a0,$h
129	ror	\$`$Sigma0[2]-$Sigma0[1]`,$a0
130	or	$c,$a1			# a|c
131
132	xor	$a0,$h			# h=Sigma0(a)
133	and	$c,$a2			# a&c
134	add	$T1,$d			# d+=T1
135
136	and	$b,$a1			# (a|c)&b
137	add	$T1,$h			# h+=T1
138
139	or	$a2,$a1			# Maj(a,b,c)=((a|c)&b)|(a&c)
140	lea	1($round),$round	# round++
141
142	add	$a1,$h			# h+=Maj(a,b,c)
143___
144}
145
146sub ROUND_16_XX()
147{ my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
148
149$code.=<<___;
150	mov	`$SZ*(($i+1)&0xf)`(%rsp),$a0
151	mov	`$SZ*(($i+14)&0xf)`(%rsp),$T1
152
153	mov	$a0,$a2
154
155	shr	\$$sigma0[2],$a0
156	ror	\$$sigma0[0],$a2
157
158	xor	$a2,$a0
159	ror	\$`$sigma0[1]-$sigma0[0]`,$a2
160
161	xor	$a2,$a0			# sigma0(X[(i+1)&0xf])
162	mov	$T1,$a1
163
164	shr	\$$sigma1[2],$T1
165	ror	\$$sigma1[0],$a1
166
167	xor	$a1,$T1
168	ror	\$`$sigma1[1]-$sigma1[0]`,$a1
169
170	xor	$a1,$T1			# sigma1(X[(i+14)&0xf])
171
172	add	$a0,$T1
173
174	add	`$SZ*(($i+9)&0xf)`(%rsp),$T1
175
176	add	`$SZ*($i&0xf)`(%rsp),$T1
177___
178	&ROUND_00_15(@_);
179}
180
181$code=<<___;
182.text
183
184.globl	$func
185.type	$func,\@function,4
186.align	16
187$func:
188	push	%rbx
189	push	%rbp
190	push	%r12
191	push	%r13
192	push	%r14
193	push	%r15
194	mov	%rsp,%r11		# copy %rsp
195	shl	\$4,%rdx		# num*16
196	sub	\$$framesz,%rsp
197	lea	($inp,%rdx,$SZ),%rdx	# inp+num*16*$SZ
198	and	\$-64,%rsp		# align stack frame
199	mov	$ctx,$_ctx		# save ctx, 1st arg
200	mov	$inp,$_inp		# save inp, 2nd arh
201	mov	%rdx,$_end		# save end pointer, "3rd" arg
202	mov	%r11,$_rsp		# save copy of %rsp
203.Lprologue:
204
205	lea	$TABLE(%rip),$Tbl
206
207	mov	$SZ*0($ctx),$A
208	mov	$SZ*1($ctx),$B
209	mov	$SZ*2($ctx),$C
210	mov	$SZ*3($ctx),$D
211	mov	$SZ*4($ctx),$E
212	mov	$SZ*5($ctx),$F
213	mov	$SZ*6($ctx),$G
214	mov	$SZ*7($ctx),$H
215	jmp	.Lloop
216
217.align	16
218.Lloop:
219	xor	$round,$round
220___
221	for($i=0;$i<16;$i++) {
222		$code.="	mov	$SZ*$i($inp),$T1\n";
223		$code.="	bswap	$T1\n";
224		&ROUND_00_15($i,@ROT);
225		unshift(@ROT,pop(@ROT));
226	}
227$code.=<<___;
228	jmp	.Lrounds_16_xx
229.align	16
230.Lrounds_16_xx:
231___
232	for(;$i<32;$i++) {
233		&ROUND_16_XX($i,@ROT);
234		unshift(@ROT,pop(@ROT));
235	}
236
237$code.=<<___;
238	cmp	\$$rounds,$round
239	jb	.Lrounds_16_xx
240
241	mov	$_ctx,$ctx
242	lea	16*$SZ($inp),$inp
243
244	add	$SZ*0($ctx),$A
245	add	$SZ*1($ctx),$B
246	add	$SZ*2($ctx),$C
247	add	$SZ*3($ctx),$D
248	add	$SZ*4($ctx),$E
249	add	$SZ*5($ctx),$F
250	add	$SZ*6($ctx),$G
251	add	$SZ*7($ctx),$H
252
253	cmp	$_end,$inp
254
255	mov	$A,$SZ*0($ctx)
256	mov	$B,$SZ*1($ctx)
257	mov	$C,$SZ*2($ctx)
258	mov	$D,$SZ*3($ctx)
259	mov	$E,$SZ*4($ctx)
260	mov	$F,$SZ*5($ctx)
261	mov	$G,$SZ*6($ctx)
262	mov	$H,$SZ*7($ctx)
263	jb	.Lloop
264
265	mov	$_rsp,%rsi
266	mov	(%rsi),%r15
267	mov	8(%rsi),%r14
268	mov	16(%rsi),%r13
269	mov	24(%rsi),%r12
270	mov	32(%rsi),%rbp
271	mov	40(%rsi),%rbx
272	lea	48(%rsi),%rsp
273.Lepilogue:
274	ret
275.size	$func,.-$func
276___
277
278if ($SZ==4) {
279$code.=<<___;
280.align	64
281.type	$TABLE,\@object
282$TABLE:
283	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
284	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
285	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
286	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
287	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
288	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
289	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
290	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
291	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
292	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
293	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
294	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
295	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
296	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
297	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
298	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
299___
300} else {
301$code.=<<___;
302.align	64
303.type	$TABLE,\@object
304$TABLE:
305	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
306	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
307	.quad	0x3956c25bf348b538,0x59f111f1b605d019
308	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
309	.quad	0xd807aa98a3030242,0x12835b0145706fbe
310	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
311	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
312	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
313	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
314	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
315	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
316	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
317	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
318	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
319	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
320	.quad	0x06ca6351e003826f,0x142929670a0e6e70
321	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
322	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
323	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
324	.quad	0x81c2c92e47edaee6,0x92722c851482353b
325	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
326	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
327	.quad	0xd192e819d6ef5218,0xd69906245565a910
328	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
329	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
330	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
331	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
332	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
333	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
334	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
335	.quad	0x90befffa23631e28,0xa4506cebde82bde9
336	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
337	.quad	0xca273eceea26619c,0xd186b8c721c0c207
338	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
339	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
340	.quad	0x113f9804bef90dae,0x1b710b35131c471b
341	.quad	0x28db77f523047d84,0x32caab7b40c72493
342	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
343	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
344	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
345___
346}
347
348# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
349#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
350if ($win64) {
351$rec="%rcx";
352$frame="%rdx";
353$context="%r8";
354$disp="%r9";
355
356$code.=<<___;
357.extern	__imp_RtlVirtualUnwind
358.type	se_handler,\@abi-omnipotent
359.align	16
360se_handler:
361	push	%rsi
362	push	%rdi
363	push	%rbx
364	push	%rbp
365	push	%r12
366	push	%r13
367	push	%r14
368	push	%r15
369	pushfq
370	sub	\$64,%rsp
371
372	mov	120($context),%rax	# pull context->Rax
373	mov	248($context),%rbx	# pull context->Rip
374
375	lea	.Lprologue(%rip),%r10
376	cmp	%r10,%rbx		# context->Rip<.Lprologue
377	jb	.Lin_prologue
378
379	mov	152($context),%rax	# pull context->Rsp
380
381	lea	.Lepilogue(%rip),%r10
382	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
383	jae	.Lin_prologue
384
385	mov	16*$SZ+3*8(%rax),%rax	# pull $_rsp
386	lea	48(%rax),%rax
387
388	mov	-8(%rax),%rbx
389	mov	-16(%rax),%rbp
390	mov	-24(%rax),%r12
391	mov	-32(%rax),%r13
392	mov	-40(%rax),%r14
393	mov	-48(%rax),%r15
394	mov	%rbx,144($context)	# restore context->Rbx
395	mov	%rbp,160($context)	# restore context->Rbp
396	mov	%r12,216($context)	# restore context->R12
397	mov	%r13,224($context)	# restore context->R13
398	mov	%r14,232($context)	# restore context->R14
399	mov	%r15,240($context)	# restore context->R15
400
401.Lin_prologue:
402	mov	8(%rax),%rdi
403	mov	16(%rax),%rsi
404	mov	%rax,152($context)	# restore context->Rsp
405	mov	%rsi,168($context)	# restore context->Rsi
406	mov	%rdi,176($context)	# restore context->Rdi
407
408	mov	40($disp),%rdi		# disp->ContextRecord
409	mov	$context,%rsi		# context
410	mov	\$154,%ecx		# sizeof(CONTEXT)
411	.long	0xa548f3fc		# cld; rep movsq
412
413	mov	$disp,%rsi
414	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
415	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
416	mov	0(%rsi),%r8		# arg3, disp->ControlPc
417	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
418	mov	40(%rsi),%r10		# disp->ContextRecord
419	lea	56(%rsi),%r11		# &disp->HandlerData
420	lea	24(%rsi),%r12		# &disp->EstablisherFrame
421	mov	%r10,32(%rsp)		# arg5
422	mov	%r11,40(%rsp)		# arg6
423	mov	%r12,48(%rsp)		# arg7
424	mov	%rcx,56(%rsp)		# arg8, (NULL)
425	call	*__imp_RtlVirtualUnwind(%rip)
426
427	mov	\$1,%eax		# ExceptionContinueSearch
428	add	\$64,%rsp
429	popfq
430	pop	%r15
431	pop	%r14
432	pop	%r13
433	pop	%r12
434	pop	%rbp
435	pop	%rbx
436	pop	%rdi
437	pop	%rsi
438	ret
439.size	se_handler,.-se_handler
440
441.section	.pdata
442.align	4
443	.rva	.LSEH_begin_$func
444	.rva	.LSEH_end_$func
445	.rva	.LSEH_info_$func
446
447.section	.xdata
448.align	8
449.LSEH_info_$func:
450	.byte	9,0,0,0
451	.rva	se_handler
452___
453}
454
455$code =~ s/\`([^\`]*)\`/eval $1/gem;
456print $code;
457close STDOUT;
458