1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8#
9# Permission to use under GPL terms is granted.
10# ====================================================================
11
12# SHA256 block procedure for ARMv4. May 2007.
13
14# Performance is ~2x better than gcc 3.4 generated code and in "abso-
15# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
16# byte [on single-issue Xscale PXA250 core].
17
18# July 2010.
19#
20# Rescheduling for dual-issue pipeline resulted in 22% improvement on
21# Cortex A8 core and ~20 cycles per processed byte.
22
23# February 2011.
24#
25# Profiler-assisted and platform-specific optimization resulted in 16%
26# improvement on Cortex A8 core and ~15.4 cycles per processed byte.
27
28# September 2013.
29#
30# Add NEON implementation. On Cortex A8 it was measured to process one
31# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
32# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
33# code (meaning that latter performs sub-optimally, nothing was done
34# about it).
35
36# May 2014.
37#
38# Add ARMv8 code path performing at 2.0 cpb on Apple A7.
39
40while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
41open STDOUT,">$output";
42
43$ctx="r0"; $t0="r0";
44$inp="r1"; $t4="r1";
45$len="r2"; $t1="r2";
46$T1="r3"; $t3="r3";
47$A="r4";
48$B="r5";
49$C="r6";
50$D="r7";
51$E="r8";
52$F="r9";
53$G="r10";
54$H="r11";
55@V=($A,$B,$C,$D,$E,$F,$G,$H);
56$t2="r12";
57$Ktbl="r14";
58
59@Sigma0=( 2,13,22);
60@Sigma1=( 6,11,25);
61@sigma0=( 7,18, 3);
62@sigma1=(17,19,10);
63
64sub BODY_00_15 {
65my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
66
67$code.=<<___ if ($i<16);
68#if __ARM_ARCH__>=7
69 @ ldr $t1,[$inp],#4 @ $i
70# if $i==15
71 str $inp,[sp,#17*4] @ make room for $t4
72# endif
73 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
74 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
75 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
76 rev $t1,$t1
77#else
78 @ ldrb $t1,[$inp,#3] @ $i
79 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
80 ldrb $t2,[$inp,#2]
81 ldrb $t0,[$inp,#1]
82 orr $t1,$t1,$t2,lsl#8
83 ldrb $t2,[$inp],#4
84 orr $t1,$t1,$t0,lsl#16
85# if $i==15
86 str $inp,[sp,#17*4] @ make room for $t4
87# endif
88 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
89 orr $t1,$t1,$t2,lsl#24
90 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
91#endif
92___
93$code.=<<___;
94 ldr $t2,[$Ktbl],#4 @ *K256++
95 add $h,$h,$t1 @ h+=X[i]
96 str $t1,[sp,#`$i%16`*4]
97 eor $t1,$f,$g
98 add $h,$h,$t0,ror#$Sigma1[0] @ h+=Sigma1(e)
99 and $t1,$t1,$e
100 add $h,$h,$t2 @ h+=K256[i]
101 eor $t1,$t1,$g @ Ch(e,f,g)
102 eor $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
103 add $h,$h,$t1 @ h+=Ch(e,f,g)
104#if $i==31
105 and $t2,$t2,#0xff
106 cmp $t2,#0xf2 @ done?
107#endif
108#if $i<15
109# if __ARM_ARCH__>=7
110 ldr $t1,[$inp],#4 @ prefetch
111# else
112 ldrb $t1,[$inp,#3]
113# endif
114 eor $t2,$a,$b @ a^b, b^c in next round
115#else
116 ldr $t1,[sp,#`($i+2)%16`*4] @ from future BODY_16_xx
117 eor $t2,$a,$b @ a^b, b^c in next round
118 ldr $t4,[sp,#`($i+15)%16`*4] @ from future BODY_16_xx
119#endif
120 eor $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]` @ Sigma0(a)
121 and $t3,$t3,$t2 @ (b^c)&=(a^b)
122 add $d,$d,$h @ d+=h
123 eor $t3,$t3,$b @ Maj(a,b,c)
124 add $h,$h,$t0,ror#$Sigma0[0] @ h+=Sigma0(a)
125 @ add $h,$h,$t3 @ h+=Maj(a,b,c)
126___
127 ($t2,$t3)=($t3,$t2);
128}
129
130sub BODY_16_XX {
131my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
132
133$code.=<<___;
134 @ ldr $t1,[sp,#`($i+1)%16`*4] @ $i
135 @ ldr $t4,[sp,#`($i+14)%16`*4]
136 mov $t0,$t1,ror#$sigma0[0]
137 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
138 mov $t2,$t4,ror#$sigma1[0]
139 eor $t0,$t0,$t1,ror#$sigma0[1]
140 eor $t2,$t2,$t4,ror#$sigma1[1]
141 eor $t0,$t0,$t1,lsr#$sigma0[2] @ sigma0(X[i+1])
142 ldr $t1,[sp,#`($i+0)%16`*4]
143 eor $t2,$t2,$t4,lsr#$sigma1[2] @ sigma1(X[i+14])
144 ldr $t4,[sp,#`($i+9)%16`*4]
145
146 add $t2,$t2,$t0
147 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` @ from BODY_00_15
148 add $t1,$t1,$t2
149 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
150 add $t1,$t1,$t4 @ X[i]
151___
152 &BODY_00_15(@_);
153}
154
155$code=<<___;
156#ifndef __KERNEL__
157# include "arm_arch.h"
158#else
159# define __ARM_ARCH__ __LINUX_ARM_ARCH__
160# define __ARM_MAX_ARCH__ 7
161#endif
162
163.text
164#if __ARM_ARCH__<7
165.code 32
166#else
167.syntax unified
168# ifdef __thumb2__
169.thumb
170# else
171.code 32
172# endif
173#endif
174
175.type K256,%object
176.align 5
177K256:
178.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
179.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
180.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
181.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
182.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
183.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
184.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
185.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
186.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
187.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
188.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
189.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
190.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
191.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
192.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
193.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
194.size K256,.-K256
195.word 0 @ terminator
196#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
197.LOPENSSL_armcap:
198.word OPENSSL_armcap_P-sha256_block_data_order
199#endif
200.align 5
201
202.global sha256_block_data_order
203.type sha256_block_data_order,%function
204sha256_block_data_order:
205#if __ARM_ARCH__<7
206 sub r3,pc,#8 @ sha256_block_data_order
207#else
| 1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8#
9# Permission to use under GPL terms is granted.
10# ====================================================================
11
12# SHA256 block procedure for ARMv4. May 2007.
13
14# Performance is ~2x better than gcc 3.4 generated code and in "abso-
15# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
16# byte [on single-issue Xscale PXA250 core].
17
18# July 2010.
19#
20# Rescheduling for dual-issue pipeline resulted in 22% improvement on
21# Cortex A8 core and ~20 cycles per processed byte.
22
23# February 2011.
24#
25# Profiler-assisted and platform-specific optimization resulted in 16%
26# improvement on Cortex A8 core and ~15.4 cycles per processed byte.
27
28# September 2013.
29#
30# Add NEON implementation. On Cortex A8 it was measured to process one
31# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
32# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
33# code (meaning that latter performs sub-optimally, nothing was done
34# about it).
35
36# May 2014.
37#
38# Add ARMv8 code path performing at 2.0 cpb on Apple A7.
39
40while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
41open STDOUT,">$output";
42
43$ctx="r0"; $t0="r0";
44$inp="r1"; $t4="r1";
45$len="r2"; $t1="r2";
46$T1="r3"; $t3="r3";
47$A="r4";
48$B="r5";
49$C="r6";
50$D="r7";
51$E="r8";
52$F="r9";
53$G="r10";
54$H="r11";
55@V=($A,$B,$C,$D,$E,$F,$G,$H);
56$t2="r12";
57$Ktbl="r14";
58
59@Sigma0=( 2,13,22);
60@Sigma1=( 6,11,25);
61@sigma0=( 7,18, 3);
62@sigma1=(17,19,10);
63
64sub BODY_00_15 {
65my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
66
67$code.=<<___ if ($i<16);
68#if __ARM_ARCH__>=7
69 @ ldr $t1,[$inp],#4 @ $i
70# if $i==15
71 str $inp,[sp,#17*4] @ make room for $t4
72# endif
73 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
74 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
75 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
76 rev $t1,$t1
77#else
78 @ ldrb $t1,[$inp,#3] @ $i
79 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
80 ldrb $t2,[$inp,#2]
81 ldrb $t0,[$inp,#1]
82 orr $t1,$t1,$t2,lsl#8
83 ldrb $t2,[$inp],#4
84 orr $t1,$t1,$t0,lsl#16
85# if $i==15
86 str $inp,[sp,#17*4] @ make room for $t4
87# endif
88 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
89 orr $t1,$t1,$t2,lsl#24
90 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
91#endif
92___
93$code.=<<___;
94 ldr $t2,[$Ktbl],#4 @ *K256++
95 add $h,$h,$t1 @ h+=X[i]
96 str $t1,[sp,#`$i%16`*4]
97 eor $t1,$f,$g
98 add $h,$h,$t0,ror#$Sigma1[0] @ h+=Sigma1(e)
99 and $t1,$t1,$e
100 add $h,$h,$t2 @ h+=K256[i]
101 eor $t1,$t1,$g @ Ch(e,f,g)
102 eor $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
103 add $h,$h,$t1 @ h+=Ch(e,f,g)
104#if $i==31
105 and $t2,$t2,#0xff
106 cmp $t2,#0xf2 @ done?
107#endif
108#if $i<15
109# if __ARM_ARCH__>=7
110 ldr $t1,[$inp],#4 @ prefetch
111# else
112 ldrb $t1,[$inp,#3]
113# endif
114 eor $t2,$a,$b @ a^b, b^c in next round
115#else
116 ldr $t1,[sp,#`($i+2)%16`*4] @ from future BODY_16_xx
117 eor $t2,$a,$b @ a^b, b^c in next round
118 ldr $t4,[sp,#`($i+15)%16`*4] @ from future BODY_16_xx
119#endif
120 eor $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]` @ Sigma0(a)
121 and $t3,$t3,$t2 @ (b^c)&=(a^b)
122 add $d,$d,$h @ d+=h
123 eor $t3,$t3,$b @ Maj(a,b,c)
124 add $h,$h,$t0,ror#$Sigma0[0] @ h+=Sigma0(a)
125 @ add $h,$h,$t3 @ h+=Maj(a,b,c)
126___
127 ($t2,$t3)=($t3,$t2);
128}
129
130sub BODY_16_XX {
131my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
132
133$code.=<<___;
134 @ ldr $t1,[sp,#`($i+1)%16`*4] @ $i
135 @ ldr $t4,[sp,#`($i+14)%16`*4]
136 mov $t0,$t1,ror#$sigma0[0]
137 add $a,$a,$t2 @ h+=Maj(a,b,c) from the past
138 mov $t2,$t4,ror#$sigma1[0]
139 eor $t0,$t0,$t1,ror#$sigma0[1]
140 eor $t2,$t2,$t4,ror#$sigma1[1]
141 eor $t0,$t0,$t1,lsr#$sigma0[2] @ sigma0(X[i+1])
142 ldr $t1,[sp,#`($i+0)%16`*4]
143 eor $t2,$t2,$t4,lsr#$sigma1[2] @ sigma1(X[i+14])
144 ldr $t4,[sp,#`($i+9)%16`*4]
145
146 add $t2,$t2,$t0
147 eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` @ from BODY_00_15
148 add $t1,$t1,$t2
149 eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e)
150 add $t1,$t1,$t4 @ X[i]
151___
152 &BODY_00_15(@_);
153}
154
155$code=<<___;
156#ifndef __KERNEL__
157# include "arm_arch.h"
158#else
159# define __ARM_ARCH__ __LINUX_ARM_ARCH__
160# define __ARM_MAX_ARCH__ 7
161#endif
162
163.text
164#if __ARM_ARCH__<7
165.code 32
166#else
167.syntax unified
168# ifdef __thumb2__
169.thumb
170# else
171.code 32
172# endif
173#endif
174
175.type K256,%object
176.align 5
177K256:
178.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
179.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
180.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
181.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
182.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
183.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
184.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
185.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
186.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
187.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
188.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
189.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
190.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
191.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
192.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
193.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
194.size K256,.-K256
195.word 0 @ terminator
196#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
197.LOPENSSL_armcap:
198.word OPENSSL_armcap_P-sha256_block_data_order
199#endif
200.align 5
201
202.global sha256_block_data_order
203.type sha256_block_data_order,%function
204sha256_block_data_order:
205#if __ARM_ARCH__<7
206 sub r3,pc,#8 @ sha256_block_data_order
207#else
|
209#endif
210#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
211 ldr r12,.LOPENSSL_armcap
212 ldr r12,[r3,r12] @ OPENSSL_armcap_P
213 tst r12,#ARMV8_SHA256
214 bne .LARMv8
215 tst r12,#ARMV7_NEON
216 bne .LNEON
217#endif
218 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
219 stmdb sp!,{$ctx,$inp,$len,r4-r11,lr}
220 ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
221 sub $Ktbl,r3,#256+32 @ K256
222 sub sp,sp,#16*4 @ alloca(X[16])
223.Loop:
224# if __ARM_ARCH__>=7
225 ldr $t1,[$inp],#4
226# else
227 ldrb $t1,[$inp,#3]
228# endif
229 eor $t3,$B,$C @ magic
230 eor $t2,$t2,$t2
231___
232for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
233$code.=".Lrounds_16_xx:\n";
234for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
235$code.=<<___;
236#if __ARM_ARCH__>=7
237 ite eq @ Thumb2 thing, sanity check in ARM
238#endif
239 ldreq $t3,[sp,#16*4] @ pull ctx
240 bne .Lrounds_16_xx
241
242 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
243 ldr $t0,[$t3,#0]
244 ldr $t1,[$t3,#4]
245 ldr $t2,[$t3,#8]
246 add $A,$A,$t0
247 ldr $t0,[$t3,#12]
248 add $B,$B,$t1
249 ldr $t1,[$t3,#16]
250 add $C,$C,$t2
251 ldr $t2,[$t3,#20]
252 add $D,$D,$t0
253 ldr $t0,[$t3,#24]
254 add $E,$E,$t1
255 ldr $t1,[$t3,#28]
256 add $F,$F,$t2
257 ldr $inp,[sp,#17*4] @ pull inp
258 ldr $t2,[sp,#18*4] @ pull inp+len
259 add $G,$G,$t0
260 add $H,$H,$t1
261 stmia $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
262 cmp $inp,$t2
263 sub $Ktbl,$Ktbl,#256 @ rewind Ktbl
264 bne .Loop
265
266 add sp,sp,#`16+3`*4 @ destroy frame
267#if __ARM_ARCH__>=5
268 ldmia sp!,{r4-r11,pc}
269#else
270 ldmia sp!,{r4-r11,lr}
271 tst lr,#1
272 moveq pc,lr @ be binary compatible with V4, yet
273 bx lr @ interoperable with Thumb ISA:-)
274#endif
275.size sha256_block_data_order,.-sha256_block_data_order
276___
277######################################################################
278# NEON stuff
279#
280{{{
281my @X=map("q$_",(0..3));
282my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
283my $Xfer=$t4;
284my $j=0;
285
286sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
287sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
288
289sub AUTOLOAD() # thunk [simplified] x86-style perlasm
290{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
291 my $arg = pop;
292 $arg = "#$arg" if ($arg*1 eq $arg);
293 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
294}
295
296sub Xupdate()
297{ use integer;
298 my $body = shift;
299 my @insns = (&$body,&$body,&$body,&$body);
300 my ($a,$b,$c,$d,$e,$f,$g,$h);
301
302 &vext_8 ($T0,@X[0],@X[1],4); # X[1..4]
303 eval(shift(@insns));
304 eval(shift(@insns));
305 eval(shift(@insns));
306 &vext_8 ($T1,@X[2],@X[3],4); # X[9..12]
307 eval(shift(@insns));
308 eval(shift(@insns));
309 eval(shift(@insns));
310 &vshr_u32 ($T2,$T0,$sigma0[0]);
311 eval(shift(@insns));
312 eval(shift(@insns));
313 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += X[9..12]
314 eval(shift(@insns));
315 eval(shift(@insns));
316 &vshr_u32 ($T1,$T0,$sigma0[2]);
317 eval(shift(@insns));
318 eval(shift(@insns));
319 &vsli_32 ($T2,$T0,32-$sigma0[0]);
320 eval(shift(@insns));
321 eval(shift(@insns));
322 &vshr_u32 ($T3,$T0,$sigma0[1]);
323 eval(shift(@insns));
324 eval(shift(@insns));
325 &veor ($T1,$T1,$T2);
326 eval(shift(@insns));
327 eval(shift(@insns));
328 &vsli_32 ($T3,$T0,32-$sigma0[1]);
329 eval(shift(@insns));
330 eval(shift(@insns));
331 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[0]);
332 eval(shift(@insns));
333 eval(shift(@insns));
334 &veor ($T1,$T1,$T3); # sigma0(X[1..4])
335 eval(shift(@insns));
336 eval(shift(@insns));
337 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[0]);
338 eval(shift(@insns));
339 eval(shift(@insns));
340 &vshr_u32 ($T5,&Dhi(@X[3]),$sigma1[2]);
341 eval(shift(@insns));
342 eval(shift(@insns));
343 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4])
344 eval(shift(@insns));
345 eval(shift(@insns));
346 &veor ($T5,$T5,$T4);
347 eval(shift(@insns));
348 eval(shift(@insns));
349 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[1]);
350 eval(shift(@insns));
351 eval(shift(@insns));
352 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[1]);
353 eval(shift(@insns));
354 eval(shift(@insns));
355 &veor ($T5,$T5,$T4); # sigma1(X[14..15])
356 eval(shift(@insns));
357 eval(shift(@insns));
358 &vadd_i32 (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
359 eval(shift(@insns));
360 eval(shift(@insns));
361 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[0]);
362 eval(shift(@insns));
363 eval(shift(@insns));
364 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[0]);
365 eval(shift(@insns));
366 eval(shift(@insns));
367 &vshr_u32 ($T5,&Dlo(@X[0]),$sigma1[2]);
368 eval(shift(@insns));
369 eval(shift(@insns));
370 &veor ($T5,$T5,$T4);
371 eval(shift(@insns));
372 eval(shift(@insns));
373 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[1]);
374 eval(shift(@insns));
375 eval(shift(@insns));
376 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
377 eval(shift(@insns));
378 eval(shift(@insns));
379 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[1]);
380 eval(shift(@insns));
381 eval(shift(@insns));
382 &veor ($T5,$T5,$T4); # sigma1(X[16..17])
383 eval(shift(@insns));
384 eval(shift(@insns));
385 &vadd_i32 (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
386 eval(shift(@insns));
387 eval(shift(@insns));
388 &vadd_i32 ($T0,$T0,@X[0]);
389 while($#insns>=2) { eval(shift(@insns)); }
390 &vst1_32 ("{$T0}","[$Xfer,:128]!");
391 eval(shift(@insns));
392 eval(shift(@insns));
393
394 push(@X,shift(@X)); # "rotate" X[]
395}
396
397sub Xpreload()
398{ use integer;
399 my $body = shift;
400 my @insns = (&$body,&$body,&$body,&$body);
401 my ($a,$b,$c,$d,$e,$f,$g,$h);
402
403 eval(shift(@insns));
404 eval(shift(@insns));
405 eval(shift(@insns));
406 eval(shift(@insns));
407 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
408 eval(shift(@insns));
409 eval(shift(@insns));
410 eval(shift(@insns));
411 eval(shift(@insns));
412 &vrev32_8 (@X[0],@X[0]);
413 eval(shift(@insns));
414 eval(shift(@insns));
415 eval(shift(@insns));
416 eval(shift(@insns));
417 &vadd_i32 ($T0,$T0,@X[0]);
418 foreach (@insns) { eval; } # remaining instructions
419 &vst1_32 ("{$T0}","[$Xfer,:128]!");
420
421 push(@X,shift(@X)); # "rotate" X[]
422}
423
424sub body_00_15 () {
425 (
426 '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
427 '&add ($h,$h,$t1)', # h+=X[i]+K[i]
428 '&eor ($t1,$f,$g)',
429 '&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
430 '&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past
431 '&and ($t1,$t1,$e)',
432 '&eor ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e)
433 '&eor ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
434 '&eor ($t1,$t1,$g)', # Ch(e,f,g)
435 '&add ($h,$h,$t2,"ror#$Sigma1[0]")', # h+=Sigma1(e)
436 '&eor ($t2,$a,$b)', # a^b, b^c in next round
437 '&eor ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a)
438 '&add ($h,$h,$t1)', # h+=Ch(e,f,g)
439 '&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'.
440 '&ldr ($t1,"[$Ktbl]") if ($j==15);'.
441 '&ldr ($t1,"[sp,#64]") if ($j==31)',
442 '&and ($t3,$t3,$t2)', # (b^c)&=(a^b)
443 '&add ($d,$d,$h)', # d+=h
444 '&add ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
445 '&eor ($t3,$t3,$b)', # Maj(a,b,c)
446 '$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
447 )
448}
449
450$code.=<<___;
451#if __ARM_MAX_ARCH__>=7
452.arch armv7-a
453.fpu neon
454
455.global sha256_block_data_order_neon
456.type sha256_block_data_order_neon,%function
457.align 4
458sha256_block_data_order_neon:
459.LNEON:
460 stmdb sp!,{r4-r12,lr}
461
462 sub $H,sp,#16*4+16
463 adr $Ktbl,K256
464 bic $H,$H,#15 @ align for 128-bit stores
465 mov $t2,sp
466 mov sp,$H @ alloca
467 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
468
469 vld1.8 {@X[0]},[$inp]!
470 vld1.8 {@X[1]},[$inp]!
471 vld1.8 {@X[2]},[$inp]!
472 vld1.8 {@X[3]},[$inp]!
473 vld1.32 {$T0},[$Ktbl,:128]!
474 vld1.32 {$T1},[$Ktbl,:128]!
475 vld1.32 {$T2},[$Ktbl,:128]!
476 vld1.32 {$T3},[$Ktbl,:128]!
477 vrev32.8 @X[0],@X[0] @ yes, even on
478 str $ctx,[sp,#64]
479 vrev32.8 @X[1],@X[1] @ big-endian
480 str $inp,[sp,#68]
481 mov $Xfer,sp
482 vrev32.8 @X[2],@X[2]
483 str $len,[sp,#72]
484 vrev32.8 @X[3],@X[3]
485 str $t2,[sp,#76] @ save original sp
486 vadd.i32 $T0,$T0,@X[0]
487 vadd.i32 $T1,$T1,@X[1]
488 vst1.32 {$T0},[$Xfer,:128]!
489 vadd.i32 $T2,$T2,@X[2]
490 vst1.32 {$T1},[$Xfer,:128]!
491 vadd.i32 $T3,$T3,@X[3]
492 vst1.32 {$T2},[$Xfer,:128]!
493 vst1.32 {$T3},[$Xfer,:128]!
494
495 ldmia $ctx,{$A-$H}
496 sub $Xfer,$Xfer,#64
497 ldr $t1,[sp,#0]
498 eor $t2,$t2,$t2
499 eor $t3,$B,$C
500 b .L_00_48
501
502.align 4
503.L_00_48:
504___
505 &Xupdate(\&body_00_15);
506 &Xupdate(\&body_00_15);
507 &Xupdate(\&body_00_15);
508 &Xupdate(\&body_00_15);
509$code.=<<___;
510 teq $t1,#0 @ check for K256 terminator
511 ldr $t1,[sp,#0]
512 sub $Xfer,$Xfer,#64
513 bne .L_00_48
514
515 ldr $inp,[sp,#68]
516 ldr $t0,[sp,#72]
517 sub $Ktbl,$Ktbl,#256 @ rewind $Ktbl
518 teq $inp,$t0
519 it eq
520 subeq $inp,$inp,#64 @ avoid SEGV
521 vld1.8 {@X[0]},[$inp]! @ load next input block
522 vld1.8 {@X[1]},[$inp]!
523 vld1.8 {@X[2]},[$inp]!
524 vld1.8 {@X[3]},[$inp]!
525 it ne
526 strne $inp,[sp,#68]
527 mov $Xfer,sp
528___
529 &Xpreload(\&body_00_15);
530 &Xpreload(\&body_00_15);
531 &Xpreload(\&body_00_15);
532 &Xpreload(\&body_00_15);
533$code.=<<___;
534 ldr $t0,[$t1,#0]
535 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
536 ldr $t2,[$t1,#4]
537 ldr $t3,[$t1,#8]
538 ldr $t4,[$t1,#12]
539 add $A,$A,$t0 @ accumulate
540 ldr $t0,[$t1,#16]
541 add $B,$B,$t2
542 ldr $t2,[$t1,#20]
543 add $C,$C,$t3
544 ldr $t3,[$t1,#24]
545 add $D,$D,$t4
546 ldr $t4,[$t1,#28]
547 add $E,$E,$t0
548 str $A,[$t1],#4
549 add $F,$F,$t2
550 str $B,[$t1],#4
551 add $G,$G,$t3
552 str $C,[$t1],#4
553 add $H,$H,$t4
554 str $D,[$t1],#4
555 stmia $t1,{$E-$H}
556
557 ittte ne
558 movne $Xfer,sp
559 ldrne $t1,[sp,#0]
560 eorne $t2,$t2,$t2
561 ldreq sp,[sp,#76] @ restore original sp
562 itt ne
563 eorne $t3,$B,$C
564 bne .L_00_48
565
566 ldmia sp!,{r4-r12,pc}
567.size sha256_block_data_order_neon,.-sha256_block_data_order_neon
568#endif
569___
570}}}
571######################################################################
572# ARMv8 stuff
573#
574{{{
575my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
576my @MSG=map("q$_",(8..11));
577my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
578my $Ktbl="r3";
579
580$code.=<<___;
581#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
582
583# ifdef __thumb2__
584# define INST(a,b,c,d) .byte c,d|0xc,a,b
585# else
586# define INST(a,b,c,d) .byte a,b,c,d
587# endif
588
589.type sha256_block_data_order_armv8,%function
590.align 5
591sha256_block_data_order_armv8:
592.LARMv8:
593 vld1.32 {$ABCD,$EFGH},[$ctx]
594# ifdef __thumb2__
595 adr $Ktbl,.LARMv8
596 sub $Ktbl,$Ktbl,#.LARMv8-K256
597# else
598 sub $Ktbl,$Ktbl,#256+32
599# endif
600 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
601
602.Loop_v8:
603 vld1.8 {@MSG[0]-@MSG[1]},[$inp]!
604 vld1.8 {@MSG[2]-@MSG[3]},[$inp]!
605 vld1.32 {$W0},[$Ktbl]!
606 vrev32.8 @MSG[0],@MSG[0]
607 vrev32.8 @MSG[1],@MSG[1]
608 vrev32.8 @MSG[2],@MSG[2]
609 vrev32.8 @MSG[3],@MSG[3]
610 vmov $ABCD_SAVE,$ABCD @ offload
611 vmov $EFGH_SAVE,$EFGH
612 teq $inp,$len
613___
614for($i=0;$i<12;$i++) {
615$code.=<<___;
616 vld1.32 {$W1},[$Ktbl]!
617 vadd.i32 $W0,$W0,@MSG[0]
618 sha256su0 @MSG[0],@MSG[1]
619 vmov $abcd,$ABCD
620 sha256h $ABCD,$EFGH,$W0
621 sha256h2 $EFGH,$abcd,$W0
622 sha256su1 @MSG[0],@MSG[2],@MSG[3]
623___
624 ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
625}
626$code.=<<___;
627 vld1.32 {$W1},[$Ktbl]!
628 vadd.i32 $W0,$W0,@MSG[0]
629 vmov $abcd,$ABCD
630 sha256h $ABCD,$EFGH,$W0
631 sha256h2 $EFGH,$abcd,$W0
632
633 vld1.32 {$W0},[$Ktbl]!
634 vadd.i32 $W1,$W1,@MSG[1]
635 vmov $abcd,$ABCD
636 sha256h $ABCD,$EFGH,$W1
637 sha256h2 $EFGH,$abcd,$W1
638
639 vld1.32 {$W1},[$Ktbl]
640 vadd.i32 $W0,$W0,@MSG[2]
641 sub $Ktbl,$Ktbl,#256-16 @ rewind
642 vmov $abcd,$ABCD
643 sha256h $ABCD,$EFGH,$W0
644 sha256h2 $EFGH,$abcd,$W0
645
646 vadd.i32 $W1,$W1,@MSG[3]
647 vmov $abcd,$ABCD
648 sha256h $ABCD,$EFGH,$W1
649 sha256h2 $EFGH,$abcd,$W1
650
651 vadd.i32 $ABCD,$ABCD,$ABCD_SAVE
652 vadd.i32 $EFGH,$EFGH,$EFGH_SAVE
653 it ne
654 bne .Loop_v8
655
656 vst1.32 {$ABCD,$EFGH},[$ctx]
657
658 ret @ bx lr
659.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
660#endif
661___
662}}}
663$code.=<<___;
664.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
665.align 2
666#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
667.comm OPENSSL_armcap_P,4,4
668#endif
669___
670
671open SELF,$0;
672while(<SELF>) {
673 next if (/^#!/);
674 last if (!s/^#/@/ and !/^$/);
675 print;
676}
677close SELF;
678
679{ my %opcode = (
680 "sha256h" => 0xf3000c40, "sha256h2" => 0xf3100c40,
681 "sha256su0" => 0xf3ba03c0, "sha256su1" => 0xf3200c40 );
682
683 sub unsha256 {
684 my ($mnemonic,$arg)=@_;
685
686 if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
687 my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
688 |(($2&7)<<17)|(($2&8)<<4)
689 |(($3&7)<<1) |(($3&8)<<2);
690 # since ARMv7 instructions are always encoded little-endian.
691 # correct solution is to use .inst directive, but older
692 # assemblers don't implement it:-(
693 sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
694 $word&0xff,($word>>8)&0xff,
695 ($word>>16)&0xff,($word>>24)&0xff,
696 $mnemonic,$arg;
697 }
698 }
699}
700
701foreach (split($/,$code)) {
702
703 s/\`([^\`]*)\`/eval $1/geo;
704
705 s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
706
707 s/\bret\b/bx lr/go or
708 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
709
710 print $_,"\n";
711}
712
713close STDOUT; # enforce flush
| 209#endif
210#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
211 ldr r12,.LOPENSSL_armcap
212 ldr r12,[r3,r12] @ OPENSSL_armcap_P
213 tst r12,#ARMV8_SHA256
214 bne .LARMv8
215 tst r12,#ARMV7_NEON
216 bne .LNEON
217#endif
218 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
219 stmdb sp!,{$ctx,$inp,$len,r4-r11,lr}
220 ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
221 sub $Ktbl,r3,#256+32 @ K256
222 sub sp,sp,#16*4 @ alloca(X[16])
223.Loop:
224# if __ARM_ARCH__>=7
225 ldr $t1,[$inp],#4
226# else
227 ldrb $t1,[$inp,#3]
228# endif
229 eor $t3,$B,$C @ magic
230 eor $t2,$t2,$t2
231___
232for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
233$code.=".Lrounds_16_xx:\n";
234for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
235$code.=<<___;
236#if __ARM_ARCH__>=7
237 ite eq @ Thumb2 thing, sanity check in ARM
238#endif
239 ldreq $t3,[sp,#16*4] @ pull ctx
240 bne .Lrounds_16_xx
241
242 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
243 ldr $t0,[$t3,#0]
244 ldr $t1,[$t3,#4]
245 ldr $t2,[$t3,#8]
246 add $A,$A,$t0
247 ldr $t0,[$t3,#12]
248 add $B,$B,$t1
249 ldr $t1,[$t3,#16]
250 add $C,$C,$t2
251 ldr $t2,[$t3,#20]
252 add $D,$D,$t0
253 ldr $t0,[$t3,#24]
254 add $E,$E,$t1
255 ldr $t1,[$t3,#28]
256 add $F,$F,$t2
257 ldr $inp,[sp,#17*4] @ pull inp
258 ldr $t2,[sp,#18*4] @ pull inp+len
259 add $G,$G,$t0
260 add $H,$H,$t1
261 stmia $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
262 cmp $inp,$t2
263 sub $Ktbl,$Ktbl,#256 @ rewind Ktbl
264 bne .Loop
265
266 add sp,sp,#`16+3`*4 @ destroy frame
267#if __ARM_ARCH__>=5
268 ldmia sp!,{r4-r11,pc}
269#else
270 ldmia sp!,{r4-r11,lr}
271 tst lr,#1
272 moveq pc,lr @ be binary compatible with V4, yet
273 bx lr @ interoperable with Thumb ISA:-)
274#endif
275.size sha256_block_data_order,.-sha256_block_data_order
276___
277######################################################################
278# NEON stuff
279#
280{{{
281my @X=map("q$_",(0..3));
282my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
283my $Xfer=$t4;
284my $j=0;
285
286sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
287sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
288
289sub AUTOLOAD() # thunk [simplified] x86-style perlasm
290{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
291 my $arg = pop;
292 $arg = "#$arg" if ($arg*1 eq $arg);
293 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
294}
295
296sub Xupdate()
297{ use integer;
298 my $body = shift;
299 my @insns = (&$body,&$body,&$body,&$body);
300 my ($a,$b,$c,$d,$e,$f,$g,$h);
301
302 &vext_8 ($T0,@X[0],@X[1],4); # X[1..4]
303 eval(shift(@insns));
304 eval(shift(@insns));
305 eval(shift(@insns));
306 &vext_8 ($T1,@X[2],@X[3],4); # X[9..12]
307 eval(shift(@insns));
308 eval(shift(@insns));
309 eval(shift(@insns));
310 &vshr_u32 ($T2,$T0,$sigma0[0]);
311 eval(shift(@insns));
312 eval(shift(@insns));
313 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += X[9..12]
314 eval(shift(@insns));
315 eval(shift(@insns));
316 &vshr_u32 ($T1,$T0,$sigma0[2]);
317 eval(shift(@insns));
318 eval(shift(@insns));
319 &vsli_32 ($T2,$T0,32-$sigma0[0]);
320 eval(shift(@insns));
321 eval(shift(@insns));
322 &vshr_u32 ($T3,$T0,$sigma0[1]);
323 eval(shift(@insns));
324 eval(shift(@insns));
325 &veor ($T1,$T1,$T2);
326 eval(shift(@insns));
327 eval(shift(@insns));
328 &vsli_32 ($T3,$T0,32-$sigma0[1]);
329 eval(shift(@insns));
330 eval(shift(@insns));
331 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[0]);
332 eval(shift(@insns));
333 eval(shift(@insns));
334 &veor ($T1,$T1,$T3); # sigma0(X[1..4])
335 eval(shift(@insns));
336 eval(shift(@insns));
337 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[0]);
338 eval(shift(@insns));
339 eval(shift(@insns));
340 &vshr_u32 ($T5,&Dhi(@X[3]),$sigma1[2]);
341 eval(shift(@insns));
342 eval(shift(@insns));
343 &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4])
344 eval(shift(@insns));
345 eval(shift(@insns));
346 &veor ($T5,$T5,$T4);
347 eval(shift(@insns));
348 eval(shift(@insns));
349 &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[1]);
350 eval(shift(@insns));
351 eval(shift(@insns));
352 &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[1]);
353 eval(shift(@insns));
354 eval(shift(@insns));
355 &veor ($T5,$T5,$T4); # sigma1(X[14..15])
356 eval(shift(@insns));
357 eval(shift(@insns));
358 &vadd_i32 (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
359 eval(shift(@insns));
360 eval(shift(@insns));
361 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[0]);
362 eval(shift(@insns));
363 eval(shift(@insns));
364 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[0]);
365 eval(shift(@insns));
366 eval(shift(@insns));
367 &vshr_u32 ($T5,&Dlo(@X[0]),$sigma1[2]);
368 eval(shift(@insns));
369 eval(shift(@insns));
370 &veor ($T5,$T5,$T4);
371 eval(shift(@insns));
372 eval(shift(@insns));
373 &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[1]);
374 eval(shift(@insns));
375 eval(shift(@insns));
376 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
377 eval(shift(@insns));
378 eval(shift(@insns));
379 &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[1]);
380 eval(shift(@insns));
381 eval(shift(@insns));
382 &veor ($T5,$T5,$T4); # sigma1(X[16..17])
383 eval(shift(@insns));
384 eval(shift(@insns));
385 &vadd_i32 (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
386 eval(shift(@insns));
387 eval(shift(@insns));
388 &vadd_i32 ($T0,$T0,@X[0]);
389 while($#insns>=2) { eval(shift(@insns)); }
390 &vst1_32 ("{$T0}","[$Xfer,:128]!");
391 eval(shift(@insns));
392 eval(shift(@insns));
393
394 push(@X,shift(@X)); # "rotate" X[]
395}
396
397sub Xpreload()
398{ use integer;
399 my $body = shift;
400 my @insns = (&$body,&$body,&$body,&$body);
401 my ($a,$b,$c,$d,$e,$f,$g,$h);
402
403 eval(shift(@insns));
404 eval(shift(@insns));
405 eval(shift(@insns));
406 eval(shift(@insns));
407 &vld1_32 ("{$T0}","[$Ktbl,:128]!");
408 eval(shift(@insns));
409 eval(shift(@insns));
410 eval(shift(@insns));
411 eval(shift(@insns));
412 &vrev32_8 (@X[0],@X[0]);
413 eval(shift(@insns));
414 eval(shift(@insns));
415 eval(shift(@insns));
416 eval(shift(@insns));
417 &vadd_i32 ($T0,$T0,@X[0]);
418 foreach (@insns) { eval; } # remaining instructions
419 &vst1_32 ("{$T0}","[$Xfer,:128]!");
420
421 push(@X,shift(@X)); # "rotate" X[]
422}
423
424sub body_00_15 () {
425 (
426 '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
427 '&add ($h,$h,$t1)', # h+=X[i]+K[i]
428 '&eor ($t1,$f,$g)',
429 '&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
430 '&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past
431 '&and ($t1,$t1,$e)',
432 '&eor ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e)
433 '&eor ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
434 '&eor ($t1,$t1,$g)', # Ch(e,f,g)
435 '&add ($h,$h,$t2,"ror#$Sigma1[0]")', # h+=Sigma1(e)
436 '&eor ($t2,$a,$b)', # a^b, b^c in next round
437 '&eor ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a)
438 '&add ($h,$h,$t1)', # h+=Ch(e,f,g)
439 '&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'.
440 '&ldr ($t1,"[$Ktbl]") if ($j==15);'.
441 '&ldr ($t1,"[sp,#64]") if ($j==31)',
442 '&and ($t3,$t3,$t2)', # (b^c)&=(a^b)
443 '&add ($d,$d,$h)', # d+=h
444 '&add ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
445 '&eor ($t3,$t3,$b)', # Maj(a,b,c)
446 '$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
447 )
448}
449
450$code.=<<___;
451#if __ARM_MAX_ARCH__>=7
452.arch armv7-a
453.fpu neon
454
455.global sha256_block_data_order_neon
456.type sha256_block_data_order_neon,%function
457.align 4
458sha256_block_data_order_neon:
459.LNEON:
460 stmdb sp!,{r4-r12,lr}
461
462 sub $H,sp,#16*4+16
463 adr $Ktbl,K256
464 bic $H,$H,#15 @ align for 128-bit stores
465 mov $t2,sp
466 mov sp,$H @ alloca
467 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
468
469 vld1.8 {@X[0]},[$inp]!
470 vld1.8 {@X[1]},[$inp]!
471 vld1.8 {@X[2]},[$inp]!
472 vld1.8 {@X[3]},[$inp]!
473 vld1.32 {$T0},[$Ktbl,:128]!
474 vld1.32 {$T1},[$Ktbl,:128]!
475 vld1.32 {$T2},[$Ktbl,:128]!
476 vld1.32 {$T3},[$Ktbl,:128]!
477 vrev32.8 @X[0],@X[0] @ yes, even on
478 str $ctx,[sp,#64]
479 vrev32.8 @X[1],@X[1] @ big-endian
480 str $inp,[sp,#68]
481 mov $Xfer,sp
482 vrev32.8 @X[2],@X[2]
483 str $len,[sp,#72]
484 vrev32.8 @X[3],@X[3]
485 str $t2,[sp,#76] @ save original sp
486 vadd.i32 $T0,$T0,@X[0]
487 vadd.i32 $T1,$T1,@X[1]
488 vst1.32 {$T0},[$Xfer,:128]!
489 vadd.i32 $T2,$T2,@X[2]
490 vst1.32 {$T1},[$Xfer,:128]!
491 vadd.i32 $T3,$T3,@X[3]
492 vst1.32 {$T2},[$Xfer,:128]!
493 vst1.32 {$T3},[$Xfer,:128]!
494
495 ldmia $ctx,{$A-$H}
496 sub $Xfer,$Xfer,#64
497 ldr $t1,[sp,#0]
498 eor $t2,$t2,$t2
499 eor $t3,$B,$C
500 b .L_00_48
501
502.align 4
503.L_00_48:
504___
505 &Xupdate(\&body_00_15);
506 &Xupdate(\&body_00_15);
507 &Xupdate(\&body_00_15);
508 &Xupdate(\&body_00_15);
509$code.=<<___;
510 teq $t1,#0 @ check for K256 terminator
511 ldr $t1,[sp,#0]
512 sub $Xfer,$Xfer,#64
513 bne .L_00_48
514
515 ldr $inp,[sp,#68]
516 ldr $t0,[sp,#72]
517 sub $Ktbl,$Ktbl,#256 @ rewind $Ktbl
518 teq $inp,$t0
519 it eq
520 subeq $inp,$inp,#64 @ avoid SEGV
521 vld1.8 {@X[0]},[$inp]! @ load next input block
522 vld1.8 {@X[1]},[$inp]!
523 vld1.8 {@X[2]},[$inp]!
524 vld1.8 {@X[3]},[$inp]!
525 it ne
526 strne $inp,[sp,#68]
527 mov $Xfer,sp
528___
529 &Xpreload(\&body_00_15);
530 &Xpreload(\&body_00_15);
531 &Xpreload(\&body_00_15);
532 &Xpreload(\&body_00_15);
533$code.=<<___;
534 ldr $t0,[$t1,#0]
535 add $A,$A,$t2 @ h+=Maj(a,b,c) from the past
536 ldr $t2,[$t1,#4]
537 ldr $t3,[$t1,#8]
538 ldr $t4,[$t1,#12]
539 add $A,$A,$t0 @ accumulate
540 ldr $t0,[$t1,#16]
541 add $B,$B,$t2
542 ldr $t2,[$t1,#20]
543 add $C,$C,$t3
544 ldr $t3,[$t1,#24]
545 add $D,$D,$t4
546 ldr $t4,[$t1,#28]
547 add $E,$E,$t0
548 str $A,[$t1],#4
549 add $F,$F,$t2
550 str $B,[$t1],#4
551 add $G,$G,$t3
552 str $C,[$t1],#4
553 add $H,$H,$t4
554 str $D,[$t1],#4
555 stmia $t1,{$E-$H}
556
557 ittte ne
558 movne $Xfer,sp
559 ldrne $t1,[sp,#0]
560 eorne $t2,$t2,$t2
561 ldreq sp,[sp,#76] @ restore original sp
562 itt ne
563 eorne $t3,$B,$C
564 bne .L_00_48
565
566 ldmia sp!,{r4-r12,pc}
567.size sha256_block_data_order_neon,.-sha256_block_data_order_neon
568#endif
569___
570}}}
571######################################################################
572# ARMv8 stuff
573#
574{{{
575my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
576my @MSG=map("q$_",(8..11));
577my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
578my $Ktbl="r3";
579
580$code.=<<___;
581#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
582
583# ifdef __thumb2__
584# define INST(a,b,c,d) .byte c,d|0xc,a,b
585# else
586# define INST(a,b,c,d) .byte a,b,c,d
587# endif
588
589.type sha256_block_data_order_armv8,%function
590.align 5
591sha256_block_data_order_armv8:
592.LARMv8:
593 vld1.32 {$ABCD,$EFGH},[$ctx]
594# ifdef __thumb2__
595 adr $Ktbl,.LARMv8
596 sub $Ktbl,$Ktbl,#.LARMv8-K256
597# else
598 sub $Ktbl,$Ktbl,#256+32
599# endif
600 add $len,$inp,$len,lsl#6 @ len to point at the end of inp
601
602.Loop_v8:
603 vld1.8 {@MSG[0]-@MSG[1]},[$inp]!
604 vld1.8 {@MSG[2]-@MSG[3]},[$inp]!
605 vld1.32 {$W0},[$Ktbl]!
606 vrev32.8 @MSG[0],@MSG[0]
607 vrev32.8 @MSG[1],@MSG[1]
608 vrev32.8 @MSG[2],@MSG[2]
609 vrev32.8 @MSG[3],@MSG[3]
610 vmov $ABCD_SAVE,$ABCD @ offload
611 vmov $EFGH_SAVE,$EFGH
612 teq $inp,$len
613___
614for($i=0;$i<12;$i++) {
615$code.=<<___;
616 vld1.32 {$W1},[$Ktbl]!
617 vadd.i32 $W0,$W0,@MSG[0]
618 sha256su0 @MSG[0],@MSG[1]
619 vmov $abcd,$ABCD
620 sha256h $ABCD,$EFGH,$W0
621 sha256h2 $EFGH,$abcd,$W0
622 sha256su1 @MSG[0],@MSG[2],@MSG[3]
623___
624 ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
625}
626$code.=<<___;
627 vld1.32 {$W1},[$Ktbl]!
628 vadd.i32 $W0,$W0,@MSG[0]
629 vmov $abcd,$ABCD
630 sha256h $ABCD,$EFGH,$W0
631 sha256h2 $EFGH,$abcd,$W0
632
633 vld1.32 {$W0},[$Ktbl]!
634 vadd.i32 $W1,$W1,@MSG[1]
635 vmov $abcd,$ABCD
636 sha256h $ABCD,$EFGH,$W1
637 sha256h2 $EFGH,$abcd,$W1
638
639 vld1.32 {$W1},[$Ktbl]
640 vadd.i32 $W0,$W0,@MSG[2]
641 sub $Ktbl,$Ktbl,#256-16 @ rewind
642 vmov $abcd,$ABCD
643 sha256h $ABCD,$EFGH,$W0
644 sha256h2 $EFGH,$abcd,$W0
645
646 vadd.i32 $W1,$W1,@MSG[3]
647 vmov $abcd,$ABCD
648 sha256h $ABCD,$EFGH,$W1
649 sha256h2 $EFGH,$abcd,$W1
650
651 vadd.i32 $ABCD,$ABCD,$ABCD_SAVE
652 vadd.i32 $EFGH,$EFGH,$EFGH_SAVE
653 it ne
654 bne .Loop_v8
655
656 vst1.32 {$ABCD,$EFGH},[$ctx]
657
658 ret @ bx lr
659.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
660#endif
661___
662}}}
663$code.=<<___;
664.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
665.align 2
666#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
667.comm OPENSSL_armcap_P,4,4
668#endif
669___
670
671open SELF,$0;
672while(<SELF>) {
673 next if (/^#!/);
674 last if (!s/^#/@/ and !/^$/);
675 print;
676}
677close SELF;
678
679{ my %opcode = (
680 "sha256h" => 0xf3000c40, "sha256h2" => 0xf3100c40,
681 "sha256su0" => 0xf3ba03c0, "sha256su1" => 0xf3200c40 );
682
683 sub unsha256 {
684 my ($mnemonic,$arg)=@_;
685
686 if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
687 my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
688 |(($2&7)<<17)|(($2&8)<<4)
689 |(($3&7)<<1) |(($3&8)<<2);
690 # since ARMv7 instructions are always encoded little-endian.
691 # correct solution is to use .inst directive, but older
692 # assemblers don't implement it:-(
693 sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
694 $word&0xff,($word>>8)&0xff,
695 ($word>>16)&0xff,($word>>24)&0xff,
696 $mnemonic,$arg;
697 }
698 }
699}
700
701foreach (split($/,$code)) {
702
703 s/\`([^\`]*)\`/eval $1/geo;
704
705 s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
706
707 s/\bret\b/bx lr/go or
708 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
709
710 print $_,"\n";
711}
712
713close STDOUT; # enforce flush
|