Revert "Import chacha-x86.pl fix."
This reverts commit 762e1d039c1d85e4651700eed82801878a9a86bc. We no longer need
to support out < in. Better to keep the assembly aligned with upstream.
Change-Id: I345bf822953bd0e1e79ad5ab4d337dcb22e7676b
Reviewed-on: https://boringssl-review.googlesource.com/8232
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/chacha/asm/chacha-x86.pl b/crypto/chacha/asm/chacha-x86.pl
index e576029..edce43d 100755
--- a/crypto/chacha/asm/chacha-x86.pl
+++ b/crypto/chacha/asm/chacha-x86.pl
@@ -19,13 +19,13 @@
# P4 18.6/+84%
# Core2 9.56/+89% 4.83
# Westmere 9.50/+45% 3.35
-# Sandy Bridge 10.7/+47% 3.24
-# Haswell 8.22/+50% 2.89
-# Silvermont 17.8/+36% 8.53
+# Sandy Bridge 10.5/+47% 3.20
+# Haswell 8.15/+50% 2.83
+# Silvermont 17.4/+36% 8.35
# Sledgehammer 10.2/+54%
-# Bulldozer 13.5/+50% 4.39(*)
+# Bulldozer 13.4/+50% 4.38(*)
#
-# (*) Bulldozer actually executes 4xXOP code path that delivers 3.50;
+# (*) Bulldozer actually executes 4xXOP code path that delivers 3.55;
#
# Modified from upstream OpenSSL to remove the XOP code.
@@ -224,20 +224,18 @@
&xor ($a, &DWP(4*0,$b)); # xor with input
&xor ($b_,&DWP(4*4,$b));
- &mov (&DWP(4*0,"esp"),$a); # off-load for later write
+ &mov (&DWP(4*0,"esp"),$a);
&mov ($a,&wparam(0)); # load output pointer
&xor ($c, &DWP(4*8,$b));
&xor ($c_,&DWP(4*9,$b));
&xor ($d, &DWP(4*12,$b));
&xor ($d_,&DWP(4*14,$b));
- &mov (&DWP(4*4,"esp"),$b_);
- &mov ($b_,&DWP(4*0,"esp"));
- &mov (&DWP(4*8,"esp"),$c);
- &mov (&DWP(4*9,"esp"),$c_);
- &mov (&DWP(4*12,"esp"),$d);
- &mov (&DWP(4*14,"esp"),$d_);
+ &mov (&DWP(4*4,$a),$b_); # write output
+ &mov (&DWP(4*8,$a),$c);
+ &mov (&DWP(4*9,$a),$c_);
+ &mov (&DWP(4*12,$a),$d);
+ &mov (&DWP(4*14,$a),$d_);
- &mov (&DWP(4*0,$a),$b_); # write output in order
&mov ($b_,&DWP(4*1,"esp"));
&mov ($c, &DWP(4*2,"esp"));
&mov ($c_,&DWP(4*3,"esp"));
@@ -254,45 +252,35 @@
&xor ($d, &DWP(4*5,$b));
&xor ($d_,&DWP(4*6,$b));
&mov (&DWP(4*1,$a),$b_);
- &mov ($b_,&DWP(4*4,"esp"));
&mov (&DWP(4*2,$a),$c);
&mov (&DWP(4*3,$a),$c_);
- &mov (&DWP(4*4,$a),$b_);
&mov (&DWP(4*5,$a),$d);
&mov (&DWP(4*6,$a),$d_);
- &mov ($c,&DWP(4*7,"esp"));
- &mov ($d,&DWP(4*8,"esp"));
- &mov ($d_,&DWP(4*9,"esp"));
- &add ($c,&DWP(64+4*7,"esp"));
- &mov ($b_, &DWP(4*10,"esp"));
- &xor ($c,&DWP(4*7,$b));
+ &mov ($b_,&DWP(4*7,"esp"));
+ &mov ($c, &DWP(4*10,"esp"));
&mov ($c_,&DWP(4*11,"esp"));
- &mov (&DWP(4*7,$a),$c);
- &mov (&DWP(4*8,$a),$d);
- &mov (&DWP(4*9,$a),$d_);
-
- &add ($b_, &DWP(64+4*10,"esp"));
- &add ($c_,&DWP(64+4*11,"esp"));
- &xor ($b_, &DWP(4*10,$b));
- &xor ($c_,&DWP(4*11,$b));
- &mov (&DWP(4*10,$a),$b_);
- &mov (&DWP(4*11,$a),$c_);
-
- &mov ($c,&DWP(4*12,"esp"));
- &mov ($c_,&DWP(4*14,"esp"));
&mov ($d, &DWP(4*13,"esp"));
&mov ($d_,&DWP(4*15,"esp"));
+ &add ($b_,&DWP(64+4*7,"esp"));
+ &add ($c, &DWP(64+4*10,"esp"));
+ &add ($c_,&DWP(64+4*11,"esp"));
&add ($d, &DWP(64+4*13,"esp"));
&add ($d_,&DWP(64+4*15,"esp"));
+ &xor ($b_,&DWP(4*7,$b));
+ &xor ($c, &DWP(4*10,$b));
+ &xor ($c_,&DWP(4*11,$b));
&xor ($d, &DWP(4*13,$b));
&xor ($d_,&DWP(4*15,$b));
&lea ($b,&DWP(4*16,$b));
- &mov (&DWP(4*12,$a),$c);
+ &mov (&DWP(4*7,$a),$b_);
+ &mov ($b_,&DWP(4*0,"esp"));
+ &mov (&DWP(4*10,$a),$c);
&mov ($c,&wparam(2)); # len
+ &mov (&DWP(4*11,$a),$c_);
&mov (&DWP(4*13,$a),$d);
- &mov (&DWP(4*14,$a),$c_);
&mov (&DWP(4*15,$a),$d_);
+ &mov (&DWP(4*0,$a),$b_);
&lea ($a,&DWP(4*16,$a));
&sub ($c,64);
&jnz (&label("outer_loop"));
@@ -567,12 +555,12 @@
my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
- for($i=0;$i<256;$i+=64) {
- #&movdqa ($xa0,&QWP($i+16*0-128,"ebx")); # it's there
- &movdqa ($xa1,&QWP($i+16*1-128,"ebx"));
- &movdqa ($xa2,&QWP($i+16*2-128,"ebx"));
- &movdqa ($xa3,&QWP($i+16*3-128,"ebx"));
+ #&movdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
+ &movdqa ($xa1,&QWP(16*1-128,"ebx"));
+ &movdqa ($xa2,&QWP(16*2-128,"ebx"));
+ &movdqa ($xa3,&QWP(16*3-128,"ebx"));
+ for($i=0;$i<256;$i+=64) {
&paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
&paddd ($xa1,&QWP($i+16*1-128,"ebp"));
&paddd ($xa2,&QWP($i+16*2-128,"ebp"));
@@ -593,29 +581,25 @@
#($xa2,$xt2)=($xt2,$xa2);
- &movdqa (&QWP($i+16*0-128,"ebx"),$xa0);
+ &movdqu ($xt0,&QWP(64*0-128,$inp)); # load input
+ &movdqu ($xt1,&QWP(64*1-128,$inp));
+ &movdqu ($xa2,&QWP(64*2-128,$inp));
+ &movdqu ($xt3,&QWP(64*3-128,$inp));
+ &lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
+ &pxor ($xt0,$xa0);
&movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
- &movdqa (&QWP($i+16*1-128,"ebx"),$xa1);
- &movdqa (&QWP($i+16*2-128,"ebx"),$xt2);
- &movdqa (&QWP($i+16*3-128,"ebx"),$xa3);
+ &pxor ($xt1,$xa1);
+ &movdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
+ &pxor ($xt2,$xa2);
+ &movdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
+ &pxor ($xt3,$xa3);
+ &movdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
+ &movdqu (&QWP(64*0-128,$out),$xt0); # store output
+ &movdqu (&QWP(64*1-128,$out),$xt1);
+ &movdqu (&QWP(64*2-128,$out),$xt2);
+ &movdqu (&QWP(64*3-128,$out),$xt3);
+ &lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
}
- for($i=0;$i<256;$i+=64) {
- my $j = 16*($i/64);
- &movdqu ($xa0,&QWP($i+16*0-128,$inp)); # load input
- &movdqu ($xa1,&QWP($i+16*1-128,$inp));
- &movdqu ($xa2,&QWP($i+16*2-128,$inp));
- &movdqu ($xa3,&QWP($i+16*3-128,$inp));
- &pxor ($xa0,&QWP($j+64*0-128,"ebx"));
- &pxor ($xa1,&QWP($j+64*1-128,"ebx"));
- &pxor ($xa2,&QWP($j+64*2-128,"ebx"));
- &pxor ($xa3,&QWP($j+64*3-128,"ebx"));
- &movdqu (&QWP($i+16*0-128,$out),$xa0); # write output
- &movdqu (&QWP($i+16*1-128,$out),$xa1);
- &movdqu (&QWP($i+16*2-128,$out),$xa2);
- &movdqu (&QWP($i+16*3-128,$out),$xa3);
- }
- &lea ($inp,&DWP(256,$inp));
- &lea ($out,&DWP(256,$out));
&sub ($len,64*4);
&jnc (&label("outer_loop"));
