aesv8-armx.pl: avoid 32-bit lane assignment in CTR mode
ARM Cortex-A57 and Cortex-A72 cores running in 32-bit mode are affected
by silicon errata #1742098 [0] and #1655431 [1], respectively, where the
second instruction of a AES instruction pair may execute twice if an
interrupt is taken right after the first instruction consumes an input
register of which a single 32-bit lane has been updated the last time it
was modified.
Shuffle the counter assignments around a bit so that the most recent
updates when the AES instruction pair executes are 128-bit wide.
[0] ARM-EPM-049219 v23 Cortex-A57 MPCore Software Developers Errata Notice
[1] ARM-EPM-012079 v11.0 Cortex-A72 MPCore Software Developers Errata Notice
(This is imported from upstream's
409c59e8f44ae56f2587cdd8a7ce611d0e3d91d9.)
The change is applied to both 32-bit and 64-bit for simplicity, but there
was no measurable performance difference, so leaving them aligned is
easiest.
Change-Id: Ic8e5f656f59ae8c2ecb2762a066c2c9064bb34c5
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/44284
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/fipsmodule/aes/asm/aesv8-armx.pl b/crypto/fipsmodule/aes/asm/aesv8-armx.pl
index a2825ce..1a2c20b 100644
--- a/crypto/fipsmodule/aes/asm/aesv8-armx.pl
+++ b/crypto/fipsmodule/aes/asm/aesv8-armx.pl
@@ -748,20 +748,34 @@
add $key_,$key,#32
mov $cnt,$rounds
cclr $step,lo
+
+ // ARM Cortex-A57 and Cortex-A72 cores running in 32-bit mode are
+ // affected by silicon errata #1742098 [0] and #1655431 [1],
+ // respectively, where the second instruction of an aese/aesmc
+ // instruction pair may execute twice if an interrupt is taken right
+ // after the first instruction consumes an input register of which a
+ // single 32-bit lane has been updated the last time it was modified.
+ //
+ // This function uses a counter in one 32-bit lane. The vmov.32 lines
+ // could write to $dat1 and $dat2 directly, but that trips this bugs.
+ // We write to $ivec and copy to the final register as a workaround.
+ //
+ // [0] ARM-EPM-049219 v23 Cortex-A57 MPCore Software Developers Errata Notice
+ // [1] ARM-EPM-012079 v11.0 Cortex-A72 MPCore Software Developers Errata Notice
#ifndef __ARMEB__
rev $ctr, $ctr
#endif
- vorr $dat1,$dat0,$dat0
add $tctr1, $ctr, #1
- vorr $dat2,$dat0,$dat0
- add $ctr, $ctr, #2
vorr $ivec,$dat0,$dat0
rev $tctr1, $tctr1
- vmov.32 ${dat1}[3],$tctr1
+ vmov.32 ${ivec}[3],$tctr1
+ add $ctr, $ctr, #2
+ vorr $dat1,$ivec,$ivec
b.ls .Lctr32_tail
rev $tctr2, $ctr
+ vmov.32 ${ivec}[3],$tctr2
sub $len,$len,#3 // bias
- vmov.32 ${dat2}[3],$tctr2
+ vorr $dat2,$ivec,$ivec
b .Loop3x_ctr32
.align 4
@@ -788,11 +802,11 @@
aese $dat1,q8
aesmc $tmp1,$dat1
vld1.8 {$in0},[$inp],#16
- vorr $dat0,$ivec,$ivec
+ add $tctr0,$ctr,#1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
- vorr $dat1,$ivec,$ivec
+ rev $tctr0,$tctr0
aese $tmp0,q9
aesmc $tmp0,$tmp0
aese $tmp1,q9
@@ -801,8 +815,6 @@
mov $key_,$key
aese $dat2,q9
aesmc $tmp2,$dat2
- vorr $dat2,$ivec,$ivec
- add $tctr0,$ctr,#1
aese $tmp0,q12
aesmc $tmp0,$tmp0
aese $tmp1,q12
@@ -817,21 +829,26 @@
aesmc $tmp0,$tmp0
aese $tmp1,q13
aesmc $tmp1,$tmp1
+ // Note the logic to update $dat0, $dat1, and $dat1 is written to work
+ // around a bug in ARM Cortex-A57 and Cortex-A72 cores running in
+ // 32-bit mode. See the comment above.
veor $in2,$in2,$rndlast
- rev $tctr0,$tctr0
+ vmov.32 ${ivec}[3], $tctr0
aese $tmp2,q13
aesmc $tmp2,$tmp2
- vmov.32 ${dat0}[3], $tctr0
+ vorr $dat0,$ivec,$ivec
rev $tctr1,$tctr1
aese $tmp0,q14
aesmc $tmp0,$tmp0
+ vmov.32 ${ivec}[3], $tctr1
+ rev $tctr2,$ctr
aese $tmp1,q14
aesmc $tmp1,$tmp1
- vmov.32 ${dat1}[3], $tctr1
- rev $tctr2,$ctr
+ vorr $dat1,$ivec,$ivec
+ vmov.32 ${ivec}[3], $tctr2
aese $tmp2,q14
aesmc $tmp2,$tmp2
- vmov.32 ${dat2}[3], $tctr2
+ vorr $dat2,$ivec,$ivec
subs $len,$len,#3
aese $tmp0,q15
aese $tmp1,q15
