Optimize computation of H^2 from H^1 in AES-GCM Squaring requires fewer multiplications than a generic multiplication, so take advantage of this when computing H^2 from H^1. This saves a few instructions in gcm_init_vpclmulqdq_avx2 and gcm_init_vpclmulqdq_avx512. Change-Id: I528bd877a53bb2fe5fa03067eda100a81192030f Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/78287 Reviewed-by: David Benjamin <davidben@google.com> Commit-Queue: David Benjamin <davidben@google.com>
diff --git a/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl b/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl index 76b7062..23ec5da 100644 --- a/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl +++ b/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl
@@ -227,6 +227,23 @@ ___ } +# This is a specialized version of _ghash_mul that computes \a * \a, i.e. it +# squares \a. It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0. +sub _ghash_square { + my ( $a, $dst, $gfpoly, $t0, $t1 ) = @_; + return <<___; + vpclmulqdq \$0x00, $a, $a, $t0 # LO = a_L * a_L + vpclmulqdq \$0x11, $a, $a, $dst # HI = a_H * a_H + vpclmulqdq \$0x01, $t0, $gfpoly, $t1 # LO_L*(x^63 + x^62 + x^57) + vpshufd \$0x4e, $t0, $t0 # Swap halves of LO + vpxor $t0, $t1, $t1 # Fold LO into MI + vpclmulqdq \$0x01, $t1, $gfpoly, $t0 # MI_L*(x^63 + x^62 + x^57) + vpshufd \$0x4e, $t1, $t1 # Swap halves of MI + vpxor $t1, $dst, $dst # Fold MI into HI (part 1) + vpxor $t0, $dst, $dst # Fold MI into HI (part 2) +___ +} + # void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]); # # Initialize |Htable| with powers of the GHASH subkey |H|. @@ -266,8 +283,8 @@ vbroadcasti128 .Lgfpoly(%rip), $GFPOLY # Square H^1 to get H^2. - @{[ _ghash_mul $H_CUR_XMM, $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM, - $TMP0_XMM, $TMP1_XMM, $TMP2_XMM ]} + @{[ _ghash_square $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM, + $TMP0_XMM, $TMP1_XMM ]} # Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2]. vinserti128 \$1, $H_CUR_XMM, $H_INC, $H_CUR
diff --git a/crypto/fipsmodule/aes/asm/aes-gcm-avx512-x86_64.pl b/crypto/fipsmodule/aes/asm/aes-gcm-avx512-x86_64.pl index 4b98b77..36b79c3 100644 --- a/crypto/fipsmodule/aes/asm/aes-gcm-avx512-x86_64.pl +++ b/crypto/fipsmodule/aes/asm/aes-gcm-avx512-x86_64.pl
@@ -359,6 +359,22 @@ ___ } +# This is a specialized version of _ghash_mul that computes \a * \a, i.e. it +# squares \a. It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0. +sub _ghash_square { + my ( $a, $dst, $gfpoly, $t0, $t1 ) = @_; + return <<___; + vpclmulqdq \$0x00, $a, $a, $t0 # LO = a_L * a_L + vpclmulqdq \$0x11, $a, $a, $dst # HI = a_H * a_H + vpclmulqdq \$0x01, $t0, $gfpoly, $t1 # LO_L*(x^63 + x^62 + x^57) + vpshufd \$0x4e, $t0, $t0 # Swap halves of LO + vpxor $t0, $t1, $t1 # Fold LO into MI + vpclmulqdq \$0x01, $t1, $gfpoly, $t0 # MI_L*(x^63 + x^62 + x^57) + vpshufd \$0x4e, $t1, $t1 # Swap halves of MI + vpternlogd \$0x96, $t0, $t1, $dst # Fold MI into HI +___ +} + # void gcm_init_vpclmulqdq_avx512(u128 Htable[16], const uint64_t H[2]); # # Initialize |Htable| with powers of the GHASH subkey |H|. @@ -418,8 +434,8 @@ # special needs to be done to make this happen, though: H^1 * H^1 would # end up with two factors of x^-1, but the multiplication consumes one. # So the product H^2 ends up with the desired one factor of x^-1. - @{[ _ghash_mul $H_CUR_XMM, $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM, - $TMP0_XMM, $TMP1_XMM, $TMP2_XMM ]} + @{[ _ghash_square $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM, + $TMP0_XMM, $TMP1_XMM ]} # Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2]. vinserti128 \$1, $H_CUR_XMM, $H_INC_YMM, $H_CUR_YMM
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-apple.S b/gen/bcm/aes-gcm-avx2-x86_64-apple.S index 66f6356..eb2ac3b 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-apple.S +++ b/gen/bcm/aes-gcm-avx2-x86_64-apple.S
@@ -64,14 +64,10 @@ vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x01,%xmm3,%xmm3,%xmm1 - vpclmulqdq $0x10,%xmm3,%xmm3,%xmm2 - vpxor %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm6,%xmm2 + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 + vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 vpshufd $0x4e,%xmm0,%xmm0 vpxor %xmm0,%xmm1,%xmm1 - vpxor %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 vpshufd $0x4e,%xmm1,%xmm1 vpxor %xmm1,%xmm5,%xmm5
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-linux.S b/gen/bcm/aes-gcm-avx2-x86_64-linux.S index 2f2743c..018397a 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-linux.S +++ b/gen/bcm/aes-gcm-avx2-x86_64-linux.S
@@ -64,14 +64,10 @@ vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x01,%xmm3,%xmm3,%xmm1 - vpclmulqdq $0x10,%xmm3,%xmm3,%xmm2 - vpxor %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm6,%xmm2 + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 + vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 vpshufd $0x4e,%xmm0,%xmm0 vpxor %xmm0,%xmm1,%xmm1 - vpxor %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 vpshufd $0x4e,%xmm1,%xmm1 vpxor %xmm1,%xmm5,%xmm5
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-win.asm b/gen/bcm/aes-gcm-avx2-x86_64-win.asm index cdd30fb..ca1d28b 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-win.asm +++ b/gen/bcm/aes-gcm-avx2-x86_64-win.asm
@@ -75,14 +75,10 @@ vpclmulqdq xmm0,xmm3,xmm3,0x00 - vpclmulqdq xmm1,xmm3,xmm3,0x01 - vpclmulqdq xmm2,xmm3,xmm3,0x10 - vpxor xmm1,xmm1,xmm2 - vpclmulqdq xmm2,xmm6,xmm0,0x01 + vpclmulqdq xmm5,xmm3,xmm3,0x11 + vpclmulqdq xmm1,xmm6,xmm0,0x01 vpshufd xmm0,xmm0,0x4e vpxor xmm1,xmm1,xmm0 - vpxor xmm1,xmm1,xmm2 - vpclmulqdq xmm5,xmm3,xmm3,0x11 vpclmulqdq xmm0,xmm6,xmm1,0x01 vpshufd xmm1,xmm1,0x4e vpxor xmm5,xmm5,xmm1
diff --git a/gen/bcm/aes-gcm-avx512-x86_64-apple.S b/gen/bcm/aes-gcm-avx512-x86_64-apple.S index 2ab2442..874ec0a 100644 --- a/gen/bcm/aes-gcm-avx512-x86_64-apple.S +++ b/gen/bcm/aes-gcm-avx512-x86_64-apple.S
@@ -84,13 +84,10 @@ vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x01,%xmm3,%xmm3,%xmm1 - vpclmulqdq $0x10,%xmm3,%xmm3,%xmm2 - vpxord %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 - vpshufd $0x4e,%xmm0,%xmm0 - vpternlogd $0x96,%xmm2,%xmm0,%xmm1 vpclmulqdq $0x11,%xmm3,%xmm3,%xmm4 + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm1 + vpshufd $0x4e,%xmm0,%xmm0 + vpxor %xmm0,%xmm1,%xmm1 vpclmulqdq $0x01,%xmm1,%xmm5,%xmm0 vpshufd $0x4e,%xmm1,%xmm1 vpternlogd $0x96,%xmm0,%xmm1,%xmm4
diff --git a/gen/bcm/aes-gcm-avx512-x86_64-linux.S b/gen/bcm/aes-gcm-avx512-x86_64-linux.S index 4e77431..bec5e88 100644 --- a/gen/bcm/aes-gcm-avx512-x86_64-linux.S +++ b/gen/bcm/aes-gcm-avx512-x86_64-linux.S
@@ -84,13 +84,10 @@ vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x01,%xmm3,%xmm3,%xmm1 - vpclmulqdq $0x10,%xmm3,%xmm3,%xmm2 - vpxord %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 - vpshufd $0x4e,%xmm0,%xmm0 - vpternlogd $0x96,%xmm2,%xmm0,%xmm1 vpclmulqdq $0x11,%xmm3,%xmm3,%xmm4 + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm1 + vpshufd $0x4e,%xmm0,%xmm0 + vpxor %xmm0,%xmm1,%xmm1 vpclmulqdq $0x01,%xmm1,%xmm5,%xmm0 vpshufd $0x4e,%xmm1,%xmm1 vpternlogd $0x96,%xmm0,%xmm1,%xmm4
diff --git a/gen/bcm/aes-gcm-avx512-x86_64-win.asm b/gen/bcm/aes-gcm-avx512-x86_64-win.asm index 76811a0..3a86e3a 100644 --- a/gen/bcm/aes-gcm-avx512-x86_64-win.asm +++ b/gen/bcm/aes-gcm-avx512-x86_64-win.asm
@@ -91,13 +91,10 @@ vpclmulqdq xmm0,xmm3,xmm3,0x00 - vpclmulqdq xmm1,xmm3,xmm3,0x01 - vpclmulqdq xmm2,xmm3,xmm3,0x10 - vpxord xmm1,xmm1,xmm2 - vpclmulqdq xmm2,xmm5,xmm0,0x01 - vpshufd xmm0,xmm0,0x4e - vpternlogd xmm1,xmm0,xmm2,0x96 vpclmulqdq xmm4,xmm3,xmm3,0x11 + vpclmulqdq xmm1,xmm5,xmm0,0x01 + vpshufd xmm0,xmm0,0x4e + vpxor xmm1,xmm1,xmm0 vpclmulqdq xmm0,xmm5,xmm1,0x01 vpshufd xmm1,xmm1,0x4e vpternlogd xmm4,xmm1,xmm0,0x96