aes-gcm-avx10-x86_64.pl: use strict mode and sync with avx2 code
Make aes-gcm-avx10-x86_64.pl run in perl's strict mode, like
aes-gcm-avx2-x86_64.pl does. Also bring in some of the other
non-functional changes to the perl code in the avx2 version, like naming
V0-V3 as AESDATA0-AESDATA3 in the en/decryption function.
No change to the generated assembly code except whitespace.
Change-Id: I5857cfdcc881a34c971959a1962dfb181eb11450
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/77168
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl b/crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl
index 36aef35..e8cf3be 100644
--- a/crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl
+++ b/crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl
@@ -71,10 +71,14 @@
# 32, masking support, and new instructions such as vpternlogd (which can do a
# three-argument XOR). These features are very useful for AES-GCM.
-$flavour = shift;
-$output = shift;
+use strict;
+
+my $flavour = shift;
+my $output = shift;
if ( $flavour =~ /\./ ) { $output = $flavour; undef $flavour; }
+my $win64;
+my @argregs;
if ( $flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/ ) {
$win64 = 1;
@argregs = ( "%rcx", "%rdx", "%r8", "%r9" );
@@ -85,7 +89,8 @@
}
$0 =~ m/(.*[\/\\])[^\/\\]+$/;
-$dir = $1;
+my $dir = $1;
+my $xlate;
( $xlate = "${dir}x86_64-xlate.pl" and -f $xlate )
or ( $xlate = "${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate )
or die "can't locate x86_64-xlate.pl";
@@ -93,6 +98,11 @@
open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT = *OUT;
+my $g_cur_func_name;
+my $g_cur_func_uses_seh;
+my @g_cur_func_saved_gpregs;
+my @g_cur_func_saved_xmmregs;
+
sub _begin_func {
my ( $funcname, $uses_seh ) = @_;
$g_cur_func_name = $funcname;
@@ -189,7 +199,7 @@
return $code;
}
-$code = <<___;
+my $code = <<___;
.section .rodata
.align 64
@@ -229,31 +239,68 @@
# Number of powers of the hash key stored in the key struct. The powers are
# stored from highest (H^NUM_H_POWERS) to lowest (H^1).
-$NUM_H_POWERS = 16;
+my $NUM_H_POWERS = 16;
-$OFFSETOFEND_H_POWERS = $NUM_H_POWERS * 16;
+my $OFFSETOFEND_H_POWERS = $NUM_H_POWERS * 16;
# Offset to 'rounds' in AES_KEY struct
-$OFFSETOF_AES_ROUNDS = 240;
+my $OFFSETOF_AES_ROUNDS = 240;
# The current vector length in bytes
-undef $VL;
+my $VL;
+
+my (
+ $V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, $V8, $V9, $V10,
+ $V11, $V12, $V13, $V14, $V15, $V16, $V17, $V18, $V19, $V20, $V21,
+ $V22, $V23, $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31
+);
# Set the vector length in bytes. This sets the VL variable and defines
# register aliases V0-V31 that map to the ymm or zmm registers.
sub _set_veclen {
($VL) = @_;
- foreach my $i ( 0 .. 31 ) {
- if ( $VL == 32 ) {
- ${"V${i}"} = "%ymm${i}";
- }
- elsif ( $VL == 64 ) {
- ${"V${i}"} = "%zmm${i}";
- }
- else {
- die "Unsupported vector length";
- }
+ my $prefix;
+ if ( $VL == 32 ) {
+ $prefix = "%ymm";
}
+ elsif ( $VL == 64 ) {
+ $prefix = "%zmm";
+ }
+ else {
+ die "Unsupported vector length";
+ }
+ $V0 = "${prefix}0";
+ $V1 = "${prefix}1";
+ $V2 = "${prefix}2";
+ $V3 = "${prefix}3";
+ $V4 = "${prefix}4";
+ $V5 = "${prefix}5";
+ $V6 = "${prefix}6";
+ $V7 = "${prefix}7";
+ $V8 = "${prefix}8";
+ $V9 = "${prefix}9";
+ $V10 = "${prefix}10";
+ $V11 = "${prefix}11";
+ $V12 = "${prefix}12";
+ $V13 = "${prefix}13";
+ $V14 = "${prefix}14";
+ $V15 = "${prefix}15";
+ $V16 = "${prefix}16";
+ $V17 = "${prefix}17";
+ $V18 = "${prefix}18";
+ $V19 = "${prefix}19";
+ $V20 = "${prefix}20";
+ $V21 = "${prefix}21";
+ $V22 = "${prefix}22";
+ $V23 = "${prefix}23";
+ $V24 = "${prefix}24";
+ $V25 = "${prefix}25";
+ $V26 = "${prefix}26";
+ $V27 = "${prefix}27";
+ $V28 = "${prefix}28";
+ $V29 = "${prefix}29";
+ $V30 = "${prefix}30";
+ $V31 = "${prefix}31";
}
# The _ghash_mul macro multiplies the 128-bit lanes of \a by the corresponding
@@ -401,7 +448,7 @@
___
}
-$g_init_macro_expansion_count = 0;
+my $g_init_macro_expansion_count = 0;
# void gcm_init_##suffix(u128 Htable[16], const uint64_t H[2]);
#
@@ -419,7 +466,10 @@
# Function arguments
my ( $HTABLE, $H_PTR ) = @argregs[ 0 .. 1 ];
- # Additional local variables. V0-V2 and %rax are used as temporaries.
+ # Additional local variables. %rax is used as a temporary register.
+ my ( $TMP0, $TMP0_YMM, $TMP0_XMM ) = ( $V0, "%ymm0", "%xmm0" );
+ my ( $TMP1, $TMP1_YMM, $TMP1_XMM ) = ( $V1, "%ymm1", "%xmm1" );
+ my ( $TMP2, $TMP2_YMM, $TMP2_XMM ) = ( $V2, "%ymm2", "%xmm2" );
my $POWERS_PTR = "%r8";
my $RNDKEYLAST_PTR = "%r9";
my ( $H_CUR, $H_CUR_YMM, $H_CUR_XMM ) = ( "$V3", "%ymm3", "%xmm3" );
@@ -449,11 +499,11 @@
# << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit
# wide shift instruction, so instead double each of the two 64-bit
# halves and incorporate the internal carry bit into the value XOR'd.
- vpshufd \$0xd3, $H_CUR_XMM, %xmm0
- vpsrad \$31, %xmm0, %xmm0
+ vpshufd \$0xd3, $H_CUR_XMM, $TMP0_XMM
+ vpsrad \$31, $TMP0_XMM, $TMP0_XMM
vpaddq $H_CUR_XMM, $H_CUR_XMM, $H_CUR_XMM
- # H_CUR_XMM ^= xmm0 & gfpoly_and_internal_carrybit
- vpternlogd \$0x78, .Lgfpoly_and_internal_carrybit(%rip), %xmm0, $H_CUR_XMM
+ # H_CUR_XMM ^= TMP0_XMM & gfpoly_and_internal_carrybit
+ vpternlogd \$0x78, .Lgfpoly_and_internal_carrybit(%rip), $TMP0_XMM, $H_CUR_XMM
# Load the gfpoly constant.
vbroadcasti32x4 .Lgfpoly(%rip), $GFPOLY
@@ -466,7 +516,7 @@
# end up with two factors of x^-1, but the multiplication consumes one.
# So the product H^2 ends up with the desired one factor of x^-1.
@{[ _ghash_mul $H_CUR_XMM, $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM,
- "%xmm0", "%xmm1", "%xmm2" ]}
+ $TMP0_XMM, $TMP1_XMM, $TMP2_XMM ]}
# Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2].
vinserti128 \$1, $H_CUR_XMM, $H_INC_YMM, $H_CUR_YMM
@@ -478,7 +528,7 @@
# Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4].
$code .= <<___;
@{[ _ghash_mul $H_INC_YMM, $H_CUR_YMM, $H_INC_YMM, $GFPOLY_YMM,
- "%ymm0", "%ymm1", "%ymm2" ]}
+ $TMP0_YMM, $TMP1_YMM, $TMP2_YMM ]}
vinserti64x4 \$1, $H_CUR_YMM, $H_INC, $H_CUR
vshufi64x2 \$0, $H_INC, $H_INC, $H_INC
___
@@ -495,7 +545,7 @@
mov \$@{[ $NUM_H_POWERS*16/$VL - 1 ]}, %eax
.Lprecompute_next$local_label_suffix:
sub \$$VL, $POWERS_PTR
- @{[ _ghash_mul $H_INC, $H_CUR, $H_CUR, $GFPOLY, $V0, $V1, $V2 ]}
+ @{[ _ghash_mul $H_INC, $H_CUR, $H_CUR, $GFPOLY, $TMP0, $TMP1, $TMP2 ]}
vmovdqu8 $H_CUR, ($POWERS_PTR)
dec %eax
jnz .Lprecompute_next$local_label_suffix
@@ -564,7 +614,13 @@
# See _ghash_mul for the full explanation of the operations performed for each
# individual finite field multiplication and reduction.
sub _ghash_step_4x {
- my ($i) = @_;
+ my (
+ $i, $BSWAP_MASK, $GHASHDATA0, $GHASHDATA1,
+ $GHASHDATA2, $GHASHDATA3, $GHASHDATA0_XMM, $GHASHDATA1_XMM,
+ $GHASHDATA2_XMM, $GHASHDATA3_XMM, $H_POW4, $H_POW3,
+ $H_POW2, $H_POW1, $GFPOLY, $GHASHTMP0,
+ $GHASHTMP1, $GHASHTMP2, $GHASH_ACC, $GHASH_ACC_XMM
+ ) = @_;
if ( $i == 0 ) {
return <<___;
vpshufb $BSWAP_MASK, $GHASHDATA0, $GHASHDATA0
@@ -642,17 +698,17 @@
}
}
-# Update GHASH with the blocks given in GHASHDATA[0-3].
-# See _ghash_step_4x for full explanation.
+# Update GHASH with four vectors of data blocks. See _ghash_step_4x for full
+# explanation.
sub _ghash_4x {
my $code = "";
for my $i ( 0 .. 9 ) {
- $code .= _ghash_step_4x $i;
+ $code .= _ghash_step_4x $i, @_;
}
return $code;
}
-$g_ghash_macro_expansion_count = 0;
+my $g_ghash_macro_expansion_count = 0;
# void gcm_ghash_##suffix(uint8_t Xi[16], const u128 Htable[16],
# const uint8_t *in, size_t len);
@@ -661,10 +717,8 @@
# prototype. This macro supports both VL=32 and VL=64. _set_veclen must have
# been invoked with the desired length.
#
-# The generated function processes the AAD (Additional Authenticated Data) in
-# GCM. Using the key |Htable|, it updates the GHASH accumulator |Xi| with the
-# data given by |in| and |len|. On the first call, |Xi| must be all zeroes.
-# |len| must be a multiple of 16.
+# This function uses the key |Htable| to update the GHASH accumulator |Xi| with
+# the data given by |in| and |len|. |len| must be a multiple of 16.
#
# This function handles large amounts of AAD efficiently, while also keeping the
# overhead low for small amounts of AAD which is the common case. TLS uses less
@@ -674,19 +728,22 @@
my $code = "";
# Function arguments
- my ( $GHASH_ACC_PTR, $H_POWERS, $AAD, $AADLEN ) = @argregs[ 0 .. 3 ];
+ my ( $GHASH_ACC_PTR, $HTABLE, $AAD, $AADLEN ) = @argregs[ 0 .. 3 ];
# Additional local variables
- ( $GHASHDATA0, $GHASHDATA0_XMM ) = ( $V0, "%xmm0" );
- ( $GHASHDATA1, $GHASHDATA1_XMM ) = ( $V1, "%xmm1" );
- ( $GHASHDATA2, $GHASHDATA2_XMM ) = ( $V2, "%xmm2" );
- ( $GHASHDATA3, $GHASHDATA3_XMM ) = ( $V3, "%xmm3" );
- ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( $V4, "%xmm4" );
- ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( $V5, "%xmm5" );
- ( $H_POW4, $H_POW3, $H_POW2 ) = ( $V6, $V7, $V8 );
- ( $H_POW1, $H_POW1_XMM ) = ( $V9, "%xmm9" );
- ( $GFPOLY, $GFPOLY_XMM ) = ( $V10, "%xmm10" );
- ( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( $V11, $V12, $V13 );
+ my ( $GHASHDATA0, $GHASHDATA0_XMM ) = ( $V0, "%xmm0" );
+ my ( $GHASHDATA1, $GHASHDATA1_XMM ) = ( $V1, "%xmm1" );
+ my ( $GHASHDATA2, $GHASHDATA2_XMM ) = ( $V2, "%xmm2" );
+ my ( $GHASHDATA3, $GHASHDATA3_XMM ) = ( $V3, "%xmm3" );
+ my @GHASHDATA = ( $GHASHDATA0, $GHASHDATA1, $GHASHDATA2, $GHASHDATA3 );
+ my @GHASHDATA_XMM =
+ ( $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM, $GHASHDATA3_XMM );
+ my ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( $V4, "%xmm4" );
+ my ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( $V5, "%xmm5" );
+ my ( $H_POW4, $H_POW3, $H_POW2 ) = ( $V6, $V7, $V8 );
+ my ( $H_POW1, $H_POW1_XMM ) = ( $V9, "%xmm9" );
+ my ( $GFPOLY, $GFPOLY_XMM ) = ( $V10, "%xmm10" );
+ my ( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( $V11, $V12, $V13 );
$code .= <<___;
@{[ _save_xmmregs (6 .. 13) ]}
@@ -712,15 +769,15 @@
vshufi64x2 \$0, $GFPOLY, $GFPOLY, $GFPOLY
# Load the lowest set of key powers.
- vmovdqu8 $OFFSETOFEND_H_POWERS-1*$VL($H_POWERS), $H_POW1
+ vmovdqu8 $OFFSETOFEND_H_POWERS-1*$VL($HTABLE), $H_POW1
cmp \$4*$VL-1, $AADLEN
jbe .Laad_loop_1x$local_label_suffix
# AADLEN >= 4*VL. Load the higher key powers.
- vmovdqu8 $OFFSETOFEND_H_POWERS-4*$VL($H_POWERS), $H_POW4
- vmovdqu8 $OFFSETOFEND_H_POWERS-3*$VL($H_POWERS), $H_POW3
- vmovdqu8 $OFFSETOFEND_H_POWERS-2*$VL($H_POWERS), $H_POW2
+ vmovdqu8 $OFFSETOFEND_H_POWERS-4*$VL($HTABLE), $H_POW4
+ vmovdqu8 $OFFSETOFEND_H_POWERS-3*$VL($HTABLE), $H_POW3
+ vmovdqu8 $OFFSETOFEND_H_POWERS-2*$VL($HTABLE), $H_POW2
# Update GHASH with 4*VL bytes of AAD at a time.
.Laad_loop_4x$local_label_suffix:
@@ -728,7 +785,9 @@
vmovdqu8 1*$VL($AAD), $GHASHDATA1
vmovdqu8 2*$VL($AAD), $GHASHDATA2
vmovdqu8 3*$VL($AAD), $GHASHDATA3
- @{[ _ghash_4x ]}
+ @{[ _ghash_4x $BSWAP_MASK, @GHASHDATA, @GHASHDATA_XMM, $H_POW4, $H_POW3,
+ $H_POW2, $H_POW1, $GFPOLY, $GHASHTMP0, $GHASHTMP1,
+ $GHASHTMP2, $GHASH_ACC, $GHASH_ACC_XMM ]}
sub \$-4*$VL, $AAD # shorter than 'add 4*VL' when VL=32
add \$-4*$VL, $AADLEN
cmp \$4*$VL-1, $AADLEN
@@ -759,7 +818,7 @@
.Laad_blockbyblock$local_label_suffix:
test $AADLEN, $AADLEN
jz .Laad_done$local_label_suffix
- vmovdqu $OFFSETOFEND_H_POWERS-16($H_POWERS), $H_POW1_XMM
+ vmovdqu $OFFSETOFEND_H_POWERS-16($HTABLE), $H_POW1_XMM
.Laad_loop_blockbyblock$local_label_suffix:
vmovdqu ($AAD), $GHASHDATA0_XMM
vpshufb $BSWAP_MASK_XMM, $GHASHDATA0_XMM, $GHASHDATA0_XMM
@@ -778,69 +837,72 @@
return $code;
}
-# Do one non-last round of AES encryption on the counter blocks in V0-V3 using
-# the round key that has been broadcast to all 128-bit lanes of \round_key.
+# Do one non-last round of AES encryption on the counter blocks in aesdata[0-3]
+# using the round key that has been broadcast to all 128-bit lanes of round_key.
sub _vaesenc_4x {
- my ($round_key) = @_;
+ my ( $round_key, $aesdata0, $aesdata1, $aesdata2, $aesdata3 ) = @_;
return <<___;
- vaesenc $round_key, $V0, $V0
- vaesenc $round_key, $V1, $V1
- vaesenc $round_key, $V2, $V2
- vaesenc $round_key, $V3, $V3
+ vaesenc $round_key, $aesdata0, $aesdata0
+ vaesenc $round_key, $aesdata1, $aesdata1
+ vaesenc $round_key, $aesdata2, $aesdata2
+ vaesenc $round_key, $aesdata3, $aesdata3
___
}
# Start the AES encryption of four vectors of counter blocks.
sub _ctr_begin_4x {
+ my (
+ $le_ctr, $le_ctr_inc, $bswap_mask, $rndkey0,
+ $aesdata0, $aesdata1, $aesdata2, $aesdata3
+ ) = @_;
return <<___;
- # Increment LE_CTR four times to generate four vectors of little-endian
- # counter blocks, swap each to big-endian, and store them in V0-V3.
- vpshufb $BSWAP_MASK, $LE_CTR, $V0
- vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
- vpshufb $BSWAP_MASK, $LE_CTR, $V1
- vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
- vpshufb $BSWAP_MASK, $LE_CTR, $V2
- vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
- vpshufb $BSWAP_MASK, $LE_CTR, $V3
- vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
+ # Increment le_ctr four times to generate four vectors of little-endian
+ # counter blocks, swap each to big-endian, and store them in aesdata[0-3].
+ vpshufb $bswap_mask, $le_ctr, $aesdata0
+ vpaddd $le_ctr_inc, $le_ctr, $le_ctr
+ vpshufb $bswap_mask, $le_ctr, $aesdata1
+ vpaddd $le_ctr_inc, $le_ctr, $le_ctr
+ vpshufb $bswap_mask, $le_ctr, $aesdata2
+ vpaddd $le_ctr_inc, $le_ctr, $le_ctr
+ vpshufb $bswap_mask, $le_ctr, $aesdata3
+ vpaddd $le_ctr_inc, $le_ctr, $le_ctr
# AES "round zero": XOR in the zero-th round key.
- vpxord $RNDKEY0, $V0, $V0
- vpxord $RNDKEY0, $V1, $V1
- vpxord $RNDKEY0, $V2, $V2
- vpxord $RNDKEY0, $V3, $V3
+ vpxord $rndkey0, $aesdata0, $aesdata0
+ vpxord $rndkey0, $aesdata1, $aesdata1
+ vpxord $rndkey0, $aesdata2, $aesdata2
+ vpxord $rndkey0, $aesdata3, $aesdata3
___
}
-# Do the last AES round for four vectors of counter blocks V0-V3, XOR source
-# data with the resulting keystream, and write the result to DST and
-# GHASHDATA[0-3]. (Implementation differs slightly, but has the same effect.)
+# Do the last AES round for four vectors of counter blocks, XOR four vectors of
+# source data with the resulting keystream blocks, and write the result to the
+# destination buffer and ghashdata[0-3]. The implementation differs slightly as
+# it takes advantage of the property vaesenclast(key, a) ^ b ==
+# vaesenclast(key ^ b, a) to reduce latency, but it has the same effect.
sub _aesenclast_and_xor_4x {
+ my (
+ $src, $dst, $rndkeylast, $aesdata0,
+ $aesdata1, $aesdata2, $aesdata3, $ghashdata0,
+ $ghashdata1, $ghashdata2, $ghashdata3
+ ) = @_;
return <<___;
- # XOR the source data with the last round key, saving the result in
- # GHASHDATA[0-3]. This reduces latency by taking advantage of the
- # property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
- vpxord 0*$VL($SRC), $RNDKEYLAST, $GHASHDATA0
- vpxord 1*$VL($SRC), $RNDKEYLAST, $GHASHDATA1
- vpxord 2*$VL($SRC), $RNDKEYLAST, $GHASHDATA2
- vpxord 3*$VL($SRC), $RNDKEYLAST, $GHASHDATA3
-
- # Do the last AES round. This handles the XOR with the source data
- # too, as per the optimization described above.
- vaesenclast $GHASHDATA0, $V0, $GHASHDATA0
- vaesenclast $GHASHDATA1, $V1, $GHASHDATA1
- vaesenclast $GHASHDATA2, $V2, $GHASHDATA2
- vaesenclast $GHASHDATA3, $V3, $GHASHDATA3
-
- # Store the en/decrypted data to DST.
- vmovdqu8 $GHASHDATA0, 0*$VL($DST)
- vmovdqu8 $GHASHDATA1, 1*$VL($DST)
- vmovdqu8 $GHASHDATA2, 2*$VL($DST)
- vmovdqu8 $GHASHDATA3, 3*$VL($DST)
+ vpxord 0*$VL($src), $rndkeylast, $ghashdata0
+ vpxord 1*$VL($src), $rndkeylast, $ghashdata1
+ vpxord 2*$VL($src), $rndkeylast, $ghashdata2
+ vpxord 3*$VL($src), $rndkeylast, $ghashdata3
+ vaesenclast $ghashdata0, $aesdata0, $ghashdata0
+ vaesenclast $ghashdata1, $aesdata1, $ghashdata1
+ vaesenclast $ghashdata2, $aesdata2, $ghashdata2
+ vaesenclast $ghashdata3, $aesdata3, $ghashdata3
+ vmovdqu8 $ghashdata0, 0*$VL($dst)
+ vmovdqu8 $ghashdata1, 1*$VL($dst)
+ vmovdqu8 $ghashdata2, 2*$VL($dst)
+ vmovdqu8 $ghashdata3, 3*$VL($dst)
___
}
-$g_update_macro_expansion_count = 0;
+my $g_update_macro_expansion_count = 0;
# void aes_gcm_{enc,dec}_update_##suffix(const uint8_t *in, uint8_t *out,
# size_t len, const AES_KEY *key,
@@ -868,60 +930,64 @@
# 32-bit word of the counter is incremented, following the GCM standard.
sub _aes_gcm_update {
my $local_label_suffix = "__func" . ++$g_update_macro_expansion_count;
-
- my ($enc) = @_;
-
- my $code = "";
+ my ($enc) = @_;
+ my $code = "";
# Function arguments
- ( $SRC, $DST, $DATALEN, $AESKEY, $BE_CTR_PTR, $H_POWERS, $GHASH_ACC_PTR ) =
- $win64
+ my ( $SRC, $DST, $DATALEN, $AESKEY, $BE_CTR_PTR, $HTABLE, $GHASH_ACC_PTR )
+ = $win64
? ( @argregs[ 0 .. 3 ], "%rsi", "%rdi", "%r12" )
: ( @argregs[ 0 .. 5 ], "%r12" );
- # Additional local variables
-
+ # Additional local variables.
# %rax, %k1, and %k2 are used as temporary registers. BE_CTR_PTR is
# also available as a temporary register after the counter is loaded.
# AES key length in bytes
- ( $AESKEYLEN, $AESKEYLEN64 ) = ( "%r10d", "%r10" );
+ my ( $AESKEYLEN, $AESKEYLEN64 ) = ( "%r10d", "%r10" );
# Pointer to the last AES round key for the chosen AES variant
- $RNDKEYLAST_PTR = "%r11";
+ my $RNDKEYLAST_PTR = "%r11";
- # In the main loop, V0-V3 are used as AES input and output. Elsewhere
- # they are used as temporary registers.
+ # AESDATA[0-3] hold the counter blocks that are being encrypted by AES.
+ my ( $AESDATA0, $AESDATA0_XMM ) = ( $V0, "%xmm0" );
+ my ( $AESDATA1, $AESDATA1_XMM ) = ( $V1, "%xmm1" );
+ my ( $AESDATA2, $AESDATA2_XMM ) = ( $V2, "%xmm2" );
+ my ( $AESDATA3, $AESDATA3_XMM ) = ( $V3, "%xmm3" );
+ my @AESDATA = ( $AESDATA0, $AESDATA1, $AESDATA2, $AESDATA3 );
# GHASHDATA[0-3] hold the ciphertext blocks and GHASH input data.
- ( $GHASHDATA0, $GHASHDATA0_XMM ) = ( $V4, "%xmm4" );
- ( $GHASHDATA1, $GHASHDATA1_XMM ) = ( $V5, "%xmm5" );
- ( $GHASHDATA2, $GHASHDATA2_XMM ) = ( $V6, "%xmm6" );
- ( $GHASHDATA3, $GHASHDATA3_XMM ) = ( $V7, "%xmm7" );
+ my ( $GHASHDATA0, $GHASHDATA0_XMM ) = ( $V4, "%xmm4" );
+ my ( $GHASHDATA1, $GHASHDATA1_XMM ) = ( $V5, "%xmm5" );
+ my ( $GHASHDATA2, $GHASHDATA2_XMM ) = ( $V6, "%xmm6" );
+ my ( $GHASHDATA3, $GHASHDATA3_XMM ) = ( $V7, "%xmm7" );
+ my @GHASHDATA = ( $GHASHDATA0, $GHASHDATA1, $GHASHDATA2, $GHASHDATA3 );
+ my @GHASHDATA_XMM =
+ ( $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM, $GHASHDATA3_XMM );
# BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values
# using vpshufb, copied to all 128-bit lanes.
- ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( $V8, "%xmm8" );
+ my ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( $V8, "%xmm8" );
# RNDKEY temporarily holds the next AES round key.
- $RNDKEY = $V9;
+ my $RNDKEY = $V9;
# GHASH_ACC is the accumulator variable for GHASH. When fully reduced,
# only the lowest 128-bit lane can be nonzero. When not fully reduced,
# more than one lane may be used, and they need to be XOR'd together.
- ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( $V10, "%xmm10" );
+ my ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( $V10, "%xmm10" );
# LE_CTR_INC is the vector of 32-bit words that need to be added to a
# vector of little-endian counter blocks to advance it forwards.
- $LE_CTR_INC = $V11;
+ my $LE_CTR_INC = $V11;
# LE_CTR contains the next set of little-endian counter blocks.
- $LE_CTR = $V12;
+ my $LE_CTR = $V12;
# RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys,
# copied to all 128-bit lanes. RNDKEY0 is the zero-th round key,
# RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last.
- (
+ my (
$RNDKEY0, $RNDKEYLAST, $RNDKEY_M9, $RNDKEY_M8,
$RNDKEY_M7, $RNDKEY_M6, $RNDKEY_M5, $RNDKEY_M4,
$RNDKEY_M3, $RNDKEY_M2, $RNDKEY_M1
@@ -930,20 +996,27 @@
# GHASHTMP[0-2] are temporary variables used by _ghash_step_4x. These
# cannot coincide with anything used for AES encryption, since for
# performance reasons GHASH and AES encryption are interleaved.
- ( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( $V24, $V25, $V26 );
+ my ( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( $V24, $V25, $V26 );
# H_POW[4-1] contain the powers of the hash key H^(4*VL/16)...H^1. The
# descending numbering reflects the order of the key powers.
- ( $H_POW4, $H_POW3, $H_POW2, $H_POW1 ) = ( $V27, $V28, $V29, $V30 );
+ my ( $H_POW4, $H_POW3, $H_POW2, $H_POW1 ) = ( $V27, $V28, $V29, $V30 );
# GFPOLY contains the .Lgfpoly constant, copied to all 128-bit lanes.
- $GFPOLY = $V31;
+ my $GFPOLY = $V31;
+
+ my @ghash_4x_args = (
+ $BSWAP_MASK, @GHASHDATA, @GHASHDATA_XMM, $H_POW4,
+ $H_POW3, $H_POW2, $H_POW1, $GFPOLY,
+ $GHASHTMP0, $GHASHTMP1, $GHASHTMP2, $GHASH_ACC,
+ $GHASH_ACC_XMM
+ );
if ($win64) {
$code .= <<___;
- @{[ _save_gpregs $BE_CTR_PTR, $H_POWERS, $GHASH_ACC_PTR ]}
+ @{[ _save_gpregs $BE_CTR_PTR, $HTABLE, $GHASH_ACC_PTR ]}
mov 64(%rsp), $BE_CTR_PTR # arg5
- mov 72(%rsp), $H_POWERS # arg6
+ mov 72(%rsp), $HTABLE # arg6
mov 80(%rsp), $GHASH_ACC_PTR # arg7
@{[ _save_xmmregs (6 .. 15) ]}
.seh_endprologue
@@ -1000,10 +1073,10 @@
jbe .Lcrypt_loop_4x_done$local_label_suffix
# Load powers of the hash key.
- vmovdqu8 $OFFSETOFEND_H_POWERS-4*$VL($H_POWERS), $H_POW4
- vmovdqu8 $OFFSETOFEND_H_POWERS-3*$VL($H_POWERS), $H_POW3
- vmovdqu8 $OFFSETOFEND_H_POWERS-2*$VL($H_POWERS), $H_POW2
- vmovdqu8 $OFFSETOFEND_H_POWERS-1*$VL($H_POWERS), $H_POW1
+ vmovdqu8 $OFFSETOFEND_H_POWERS-4*$VL($HTABLE), $H_POW4
+ vmovdqu8 $OFFSETOFEND_H_POWERS-3*$VL($HTABLE), $H_POW3
+ vmovdqu8 $OFFSETOFEND_H_POWERS-2*$VL($HTABLE), $H_POW2
+ vmovdqu8 $OFFSETOFEND_H_POWERS-1*$VL($HTABLE), $H_POW1
___
# Main loop: en/decrypt and hash 4 vectors at a time.
@@ -1025,15 +1098,15 @@
$code .= <<___;
# Encrypt the first 4 vectors of plaintext blocks. Leave the resulting
# ciphertext in GHASHDATA[0-3] for GHASH.
- @{[ _ctr_begin_4x ]}
+ @{[ _ctr_begin_4x $LE_CTR, $LE_CTR_INC, $BSWAP_MASK, $RNDKEY0, @AESDATA ]}
lea 16($AESKEY), %rax
.Lvaesenc_loop_first_4_vecs$local_label_suffix:
vbroadcasti32x4 (%rax), $RNDKEY
- @{[ _vaesenc_4x $RNDKEY ]}
+ @{[ _vaesenc_4x $RNDKEY, @AESDATA ]}
add \$16, %rax
cmp %rax, $RNDKEYLAST_PTR
jne .Lvaesenc_loop_first_4_vecs$local_label_suffix
- @{[ _aesenclast_and_xor_4x ]}
+ @{[ _aesenclast_and_xor_4x $SRC, $DST, $RNDKEYLAST, @AESDATA, @GHASHDATA ]}
sub \$-4*$VL, $SRC # shorter than 'add 4*VL' when VL=32
sub \$-4*$VL, $DST
add \$-4*$VL, $DATALEN
@@ -1042,14 +1115,18 @@
___
}
- # Cache as many additional AES round keys as possible.
- for my $i ( reverse 1 .. 9 ) {
- $code .= <<___;
- vbroadcasti32x4 -$i*16($RNDKEYLAST_PTR), ${"RNDKEY_M$i"}
-___
- }
-
$code .= <<___;
+ # Cache as many additional AES round keys as possible.
+ vbroadcasti32x4 -9*16($RNDKEYLAST_PTR), $RNDKEY_M9
+ vbroadcasti32x4 -8*16($RNDKEYLAST_PTR), $RNDKEY_M8
+ vbroadcasti32x4 -7*16($RNDKEYLAST_PTR), $RNDKEY_M7
+ vbroadcasti32x4 -6*16($RNDKEYLAST_PTR), $RNDKEY_M6
+ vbroadcasti32x4 -5*16($RNDKEYLAST_PTR), $RNDKEY_M5
+ vbroadcasti32x4 -4*16($RNDKEYLAST_PTR), $RNDKEY_M4
+ vbroadcasti32x4 -3*16($RNDKEYLAST_PTR), $RNDKEY_M3
+ vbroadcasti32x4 -2*16($RNDKEYLAST_PTR), $RNDKEY_M2
+ vbroadcasti32x4 -1*16($RNDKEYLAST_PTR), $RNDKEY_M1
+
.Lcrypt_loop_4x$local_label_suffix:
___
@@ -1066,20 +1143,20 @@
$code .= <<___;
# Start the AES encryption of the counter blocks.
- @{[ _ctr_begin_4x ]}
+ @{[ _ctr_begin_4x $LE_CTR, $LE_CTR_INC, $BSWAP_MASK, $RNDKEY0, @AESDATA ]}
cmp \$24, $AESKEYLEN
jl .Laes128$local_label_suffix
je .Laes192$local_label_suffix
# AES-256
vbroadcasti32x4 -13*16($RNDKEYLAST_PTR), $RNDKEY
- @{[ _vaesenc_4x $RNDKEY ]}
+ @{[ _vaesenc_4x $RNDKEY, @AESDATA ]}
vbroadcasti32x4 -12*16($RNDKEYLAST_PTR), $RNDKEY
- @{[ _vaesenc_4x $RNDKEY ]}
+ @{[ _vaesenc_4x $RNDKEY, @AESDATA ]}
.Laes192$local_label_suffix:
vbroadcasti32x4 -11*16($RNDKEYLAST_PTR), $RNDKEY
- @{[ _vaesenc_4x $RNDKEY ]}
+ @{[ _vaesenc_4x $RNDKEY, @AESDATA ]}
vbroadcasti32x4 -10*16($RNDKEYLAST_PTR), $RNDKEY
- @{[ _vaesenc_4x $RNDKEY ]}
+ @{[ _vaesenc_4x $RNDKEY, @AESDATA ]}
.Laes128$local_label_suffix:
___
@@ -1090,17 +1167,31 @@
$code .= "prefetcht0 512+$i($SRC)\n";
}
- # Finish the AES encryption of the counter blocks in V0-V3, interleaved
- # with the GHASH update of the ciphertext blocks in GHASHDATA[0-3].
- for my $i ( reverse 1 .. 9 ) {
- $code .= <<___;
- @{[ _ghash_step_4x (9 - $i) ]}
- @{[ _vaesenc_4x ${"RNDKEY_M$i"} ]}
-___
- }
$code .= <<___;
- @{[ _ghash_step_4x 9 ]}
- @{[ _aesenclast_and_xor_4x ]}
+ # Finish the AES encryption of the counter blocks in AESDATA[0-3],
+ # interleaved with the GHASH update of the ciphertext blocks in
+ # GHASHDATA[0-3].
+ @{[ _ghash_step_4x 0, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M9, @AESDATA ]}
+ @{[ _ghash_step_4x 1, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M8, @AESDATA ]}
+ @{[ _ghash_step_4x 2, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M7, @AESDATA ]}
+ @{[ _ghash_step_4x 3, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M6, @AESDATA ]}
+ @{[ _ghash_step_4x 4, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M5, @AESDATA ]}
+ @{[ _ghash_step_4x 5, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M4, @AESDATA ]}
+ @{[ _ghash_step_4x 6, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M3, @AESDATA ]}
+ @{[ _ghash_step_4x 7, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M2, @AESDATA ]}
+ @{[ _ghash_step_4x 8, @ghash_4x_args ]}
+ @{[ _vaesenc_4x $RNDKEY_M1, @AESDATA ]}
+
+ @{[ _ghash_step_4x 9, @ghash_4x_args ]}
+ @{[ _aesenclast_and_xor_4x $SRC, $DST, $RNDKEYLAST, @AESDATA, @GHASHDATA ]}
sub \$-4*$VL, $SRC # shorter than 'add 4*VL' when VL=32
sub \$-4*$VL, $DST
add \$-4*$VL, $DATALEN
@@ -1113,7 +1204,7 @@
# Update GHASH with the last set of ciphertext blocks.
$code .= <<___;
.Lghash_last_ciphertext_4x$local_label_suffix:
- @{[ _ghash_4x ]}
+ @{[ _ghash_4x @ghash_4x_args ]}
___
}
@@ -1147,7 +1238,7 @@
mov $DATALEN, %rax
neg %rax
and \$-16, %rax # -round_up(DATALEN, 16)
- lea $OFFSETOFEND_H_POWERS($H_POWERS,%rax), $POWERS_PTR
+ lea $OFFSETOFEND_H_POWERS($HTABLE,%rax), $POWERS_PTR
___
# Start collecting the unreduced GHASH intermediate value LO, MI, HI.
@@ -1166,29 +1257,29 @@
# Process a full vector of length VL.
# Encrypt a vector of counter blocks.
- vpshufb $BSWAP_MASK, $LE_CTR, $V0
+ vpshufb $BSWAP_MASK, $LE_CTR, $AESDATA0
vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
- vpxord $RNDKEY0, $V0, $V0
+ vpxord $RNDKEY0, $AESDATA0, $AESDATA0
lea 16($AESKEY), %rax
.Lvaesenc_loop_tail_full_vec$local_label_suffix:
vbroadcasti32x4 (%rax), $RNDKEY
- vaesenc $RNDKEY, $V0, $V0
+ vaesenc $RNDKEY, $AESDATA0, $AESDATA0
add \$16, %rax
cmp %rax, $RNDKEYLAST_PTR
jne .Lvaesenc_loop_tail_full_vec$local_label_suffix
- vaesenclast $RNDKEYLAST, $V0, $V0
+ vaesenclast $RNDKEYLAST, $AESDATA0, $AESDATA0
# XOR the data with the vector of keystream blocks.
- vmovdqu8 ($SRC), $V1
- vpxord $V1, $V0, $V0
- vmovdqu8 $V0, ($DST)
+ vmovdqu8 ($SRC), $AESDATA1
+ vpxord $AESDATA1, $AESDATA0, $AESDATA0
+ vmovdqu8 $AESDATA0, ($DST)
# Update GHASH with the ciphertext blocks, without reducing.
vmovdqu8 ($POWERS_PTR), $H_POW1
- vpshufb $BSWAP_MASK, @{[ $enc ? $V0 : $V1 ]}, $V0
- vpxord $GHASH_ACC, $V0, $V0
- @{[ _ghash_mul_noreduce $H_POW1, $V0, $LO, $MI, $HI, $GHASHDATA3,
- $V1, $V2, $V3 ]}
+ vpshufb $BSWAP_MASK, @{[ $enc ? $AESDATA0 : $AESDATA1 ]}, $AESDATA0
+ vpxord $GHASH_ACC, $AESDATA0, $AESDATA0
+ @{[ _ghash_mul_noreduce $H_POW1, $AESDATA0, $LO, $MI, $HI,
+ $GHASHDATA3, $AESDATA1, $AESDATA2, $AESDATA3 ]}
vpxor $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
add \$$VL, $POWERS_PTR
@@ -1217,21 +1308,21 @@
# Encrypt one last vector of counter blocks. This does not need to be
# masked. The counter does not need to be incremented here.
- vpshufb $BSWAP_MASK, $LE_CTR, $V0
- vpxord $RNDKEY0, $V0, $V0
+ vpshufb $BSWAP_MASK, $LE_CTR, $AESDATA0
+ vpxord $RNDKEY0, $AESDATA0, $AESDATA0
lea 16($AESKEY), %rax
.Lvaesenc_loop_tail_partialvec$local_label_suffix:
vbroadcasti32x4 (%rax), $RNDKEY
- vaesenc $RNDKEY, $V0, $V0
+ vaesenc $RNDKEY, $AESDATA0, $AESDATA0
add \$16, %rax
cmp %rax, $RNDKEYLAST_PTR
jne .Lvaesenc_loop_tail_partialvec$local_label_suffix
- vaesenclast $RNDKEYLAST, $V0, $V0
+ vaesenclast $RNDKEYLAST, $AESDATA0, $AESDATA0
# XOR the data with the appropriate number of keystream bytes.
- vmovdqu8 ($SRC), $V1\{%k1}{z}
- vpxord $V1, $V0, $V0
- vmovdqu8 $V0, ($DST){%k1}
+ vmovdqu8 ($SRC), $AESDATA1\{%k1}{z}
+ vpxord $AESDATA1, $AESDATA0, $AESDATA0
+ vmovdqu8 $AESDATA0, ($DST){%k1}
# Update GHASH with the ciphertext block(s), without reducing.
#
@@ -1246,17 +1337,17 @@
# they're multiplied with are also all-zeroes. Therefore they just add
# 0 * 0 = 0 to the final GHASH result, which makes no difference.
vmovdqu8 ($POWERS_PTR), $H_POW1\{%k2}{z}
- @{[ $enc ? "vmovdqu8 $V0, $V1\{%k1}{z}" : "" ]}
- vpshufb $BSWAP_MASK, $V1, $V0
- vpxord $GHASH_ACC, $V0, $V0
- @{[ _ghash_mul_noreduce $H_POW1, $V0, $LO, $MI, $HI, $GHASHDATA3,
- $V1, $V2, $V3 ]}
+ @{[ $enc ? "vmovdqu8 $AESDATA0, $AESDATA1\{%k1}{z}" : "" ]}
+ vpshufb $BSWAP_MASK, $AESDATA1, $AESDATA0
+ vpxord $GHASH_ACC, $AESDATA0, $AESDATA0
+ @{[ _ghash_mul_noreduce $H_POW1, $AESDATA0, $LO, $MI, $HI,
+ $GHASHDATA3, $AESDATA1, $AESDATA2, $AESDATA3 ]}
.Lreduce$local_label_suffix:
# Finally, do the GHASH reduction.
- @{[ _ghash_reduce $LO, $MI, $HI, $GFPOLY, $V0 ]}
+ @{[ _ghash_reduce $LO, $MI, $HI, $GFPOLY, $AESDATA0 ]}
@{[ _horizontal_xor $HI, $HI_XMM, $GHASH_ACC_XMM,
- "%xmm0", "%xmm1", "%xmm2" ]}
+ $AESDATA0_XMM, $AESDATA1_XMM, $AESDATA2_XMM ]}
.Ldone$local_label_suffix:
# Store the updated GHASH accumulator back to memory.
@@ -1271,7 +1362,7 @@
# void gcm_gmult_vpclmulqdq_avx10(uint8_t Xi[16], const u128 Htable[16]);
$code .= _begin_func "gcm_gmult_vpclmulqdq_avx10", 1;
{
- my ( $GHASH_ACC_PTR, $H_POWERS ) = @argregs[ 0 .. 1 ];
+ my ( $GHASH_ACC_PTR, $HTABLE ) = @argregs[ 0 .. 1 ];
my ( $GHASH_ACC, $BSWAP_MASK, $H_POW1, $GFPOLY, $T0, $T1, $T2 ) =
map( "%xmm$_", ( 0 .. 6 ) );
@@ -1281,7 +1372,7 @@
vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC
vmovdqu .Lbswap_mask(%rip), $BSWAP_MASK
- vmovdqu $OFFSETOFEND_H_POWERS-16($H_POWERS), $H_POW1
+ vmovdqu $OFFSETOFEND_H_POWERS-16($HTABLE), $H_POW1
vmovdqu .Lgfpoly(%rip), $GFPOLY
vpshufb $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC
diff --git a/gen/bcm/aes-gcm-avx10-x86_64-apple.S b/gen/bcm/aes-gcm-avx10-x86_64-apple.S
index be66605..a7ec87e 100644
--- a/gen/bcm/aes-gcm-avx10-x86_64-apple.S
+++ b/gen/bcm/aes-gcm-avx10-x86_64-apple.S
@@ -428,22 +428,14 @@
addq $16,%rax
cmpq %rax,%r11
jne L$vaesenc_loop_first_4_vecs__func1
-
-
-
vpxord 0(%rdi),%zmm14,%zmm4
vpxord 64(%rdi),%zmm14,%zmm5
vpxord 128(%rdi),%zmm14,%zmm6
vpxord 192(%rdi),%zmm14,%zmm7
-
-
-
vaesenclast %zmm4,%zmm0,%zmm4
vaesenclast %zmm5,%zmm1,%zmm5
vaesenclast %zmm6,%zmm2,%zmm6
vaesenclast %zmm7,%zmm3,%zmm7
-
-
vmovdqu8 %zmm4,0(%rsi)
vmovdqu8 %zmm5,64(%rsi)
vmovdqu8 %zmm6,128(%rsi)
@@ -454,6 +446,7 @@
addq $-256,%rdx
cmpq $256-1,%rdx
jbe L$ghash_last_ciphertext_4x__func1
+
vbroadcasti32x4 -144(%r11),%zmm15
vbroadcasti32x4 -128(%r11),%zmm16
vbroadcasti32x4 -112(%r11),%zmm17
@@ -463,6 +456,7 @@
vbroadcasti32x4 -48(%r11),%zmm21
vbroadcasti32x4 -32(%r11),%zmm22
vbroadcasti32x4 -16(%r11),%zmm23
+
L$crypt_loop_4x__func1:
@@ -516,6 +510,9 @@
prefetcht0 512+64(%rdi)
prefetcht0 512+128(%rdi)
prefetcht0 512+192(%rdi)
+
+
+
vpshufb %zmm8,%zmm4,%zmm4
vpxord %zmm10,%zmm4,%zmm4
vpshufb %zmm8,%zmm5,%zmm5
@@ -605,28 +602,21 @@
vaesenc %zmm23,%zmm2,%zmm2
vaesenc %zmm23,%zmm3,%zmm3
+
vextracti32x4 $1,%zmm10,%xmm4
vextracti32x4 $2,%zmm10,%xmm5
vextracti32x4 $3,%zmm10,%xmm6
vpxord %xmm4,%xmm10,%xmm10
vpternlogd $0x96,%xmm5,%xmm6,%xmm10
-
-
-
vpxord 0(%rdi),%zmm14,%zmm4
vpxord 64(%rdi),%zmm14,%zmm5
vpxord 128(%rdi),%zmm14,%zmm6
vpxord 192(%rdi),%zmm14,%zmm7
-
-
-
vaesenclast %zmm4,%zmm0,%zmm4
vaesenclast %zmm5,%zmm1,%zmm5
vaesenclast %zmm6,%zmm2,%zmm6
vaesenclast %zmm7,%zmm3,%zmm7
-
-
vmovdqu8 %zmm4,0(%rsi)
vmovdqu8 %zmm5,64(%rsi)
vmovdqu8 %zmm6,128(%rsi)
@@ -895,6 +885,7 @@
vmovdqu8 256-192(%r9),%zmm28
vmovdqu8 256-128(%r9),%zmm29
vmovdqu8 256-64(%r9),%zmm30
+
vbroadcasti32x4 -144(%r11),%zmm15
vbroadcasti32x4 -128(%r11),%zmm16
vbroadcasti32x4 -112(%r11),%zmm17
@@ -904,6 +895,7 @@
vbroadcasti32x4 -48(%r11),%zmm21
vbroadcasti32x4 -32(%r11),%zmm22
vbroadcasti32x4 -16(%r11),%zmm23
+
L$crypt_loop_4x__func2:
vmovdqu8 0(%rdi),%zmm4
vmovdqu8 64(%rdi),%zmm5
@@ -961,6 +953,9 @@
prefetcht0 512+64(%rdi)
prefetcht0 512+128(%rdi)
prefetcht0 512+192(%rdi)
+
+
+
vpshufb %zmm8,%zmm4,%zmm4
vpxord %zmm10,%zmm4,%zmm4
vpshufb %zmm8,%zmm5,%zmm5
@@ -1050,28 +1045,21 @@
vaesenc %zmm23,%zmm2,%zmm2
vaesenc %zmm23,%zmm3,%zmm3
+
vextracti32x4 $1,%zmm10,%xmm4
vextracti32x4 $2,%zmm10,%xmm5
vextracti32x4 $3,%zmm10,%xmm6
vpxord %xmm4,%xmm10,%xmm10
vpternlogd $0x96,%xmm5,%xmm6,%xmm10
-
-
-
vpxord 0(%rdi),%zmm14,%zmm4
vpxord 64(%rdi),%zmm14,%zmm5
vpxord 128(%rdi),%zmm14,%zmm6
vpxord 192(%rdi),%zmm14,%zmm7
-
-
-
vaesenclast %zmm4,%zmm0,%zmm4
vaesenclast %zmm5,%zmm1,%zmm5
vaesenclast %zmm6,%zmm2,%zmm6
vaesenclast %zmm7,%zmm3,%zmm7
-
-
vmovdqu8 %zmm4,0(%rsi)
vmovdqu8 %zmm5,64(%rsi)
vmovdqu8 %zmm6,128(%rsi)
diff --git a/gen/bcm/aes-gcm-avx10-x86_64-linux.S b/gen/bcm/aes-gcm-avx10-x86_64-linux.S
index b525623..0ffc7c7 100644
--- a/gen/bcm/aes-gcm-avx10-x86_64-linux.S
+++ b/gen/bcm/aes-gcm-avx10-x86_64-linux.S
@@ -430,22 +430,14 @@
addq $16,%rax
cmpq %rax,%r11
jne .Lvaesenc_loop_first_4_vecs__func1
-
-
-
vpxord 0(%rdi),%zmm14,%zmm4
vpxord 64(%rdi),%zmm14,%zmm5
vpxord 128(%rdi),%zmm14,%zmm6
vpxord 192(%rdi),%zmm14,%zmm7
-
-
-
vaesenclast %zmm4,%zmm0,%zmm4
vaesenclast %zmm5,%zmm1,%zmm5
vaesenclast %zmm6,%zmm2,%zmm6
vaesenclast %zmm7,%zmm3,%zmm7
-
-
vmovdqu8 %zmm4,0(%rsi)
vmovdqu8 %zmm5,64(%rsi)
vmovdqu8 %zmm6,128(%rsi)
@@ -456,6 +448,7 @@
addq $-256,%rdx
cmpq $256-1,%rdx
jbe .Lghash_last_ciphertext_4x__func1
+
vbroadcasti32x4 -144(%r11),%zmm15
vbroadcasti32x4 -128(%r11),%zmm16
vbroadcasti32x4 -112(%r11),%zmm17
@@ -465,6 +458,7 @@
vbroadcasti32x4 -48(%r11),%zmm21
vbroadcasti32x4 -32(%r11),%zmm22
vbroadcasti32x4 -16(%r11),%zmm23
+
.Lcrypt_loop_4x__func1:
@@ -518,6 +512,9 @@
prefetcht0 512+64(%rdi)
prefetcht0 512+128(%rdi)
prefetcht0 512+192(%rdi)
+
+
+
vpshufb %zmm8,%zmm4,%zmm4
vpxord %zmm10,%zmm4,%zmm4
vpshufb %zmm8,%zmm5,%zmm5
@@ -607,28 +604,21 @@
vaesenc %zmm23,%zmm2,%zmm2
vaesenc %zmm23,%zmm3,%zmm3
+
vextracti32x4 $1,%zmm10,%xmm4
vextracti32x4 $2,%zmm10,%xmm5
vextracti32x4 $3,%zmm10,%xmm6
vpxord %xmm4,%xmm10,%xmm10
vpternlogd $0x96,%xmm5,%xmm6,%xmm10
-
-
-
vpxord 0(%rdi),%zmm14,%zmm4
vpxord 64(%rdi),%zmm14,%zmm5
vpxord 128(%rdi),%zmm14,%zmm6
vpxord 192(%rdi),%zmm14,%zmm7
-
-
-
vaesenclast %zmm4,%zmm0,%zmm4
vaesenclast %zmm5,%zmm1,%zmm5
vaesenclast %zmm6,%zmm2,%zmm6
vaesenclast %zmm7,%zmm3,%zmm7
-
-
vmovdqu8 %zmm4,0(%rsi)
vmovdqu8 %zmm5,64(%rsi)
vmovdqu8 %zmm6,128(%rsi)
@@ -899,6 +889,7 @@
vmovdqu8 256-192(%r9),%zmm28
vmovdqu8 256-128(%r9),%zmm29
vmovdqu8 256-64(%r9),%zmm30
+
vbroadcasti32x4 -144(%r11),%zmm15
vbroadcasti32x4 -128(%r11),%zmm16
vbroadcasti32x4 -112(%r11),%zmm17
@@ -908,6 +899,7 @@
vbroadcasti32x4 -48(%r11),%zmm21
vbroadcasti32x4 -32(%r11),%zmm22
vbroadcasti32x4 -16(%r11),%zmm23
+
.Lcrypt_loop_4x__func2:
vmovdqu8 0(%rdi),%zmm4
vmovdqu8 64(%rdi),%zmm5
@@ -965,6 +957,9 @@
prefetcht0 512+64(%rdi)
prefetcht0 512+128(%rdi)
prefetcht0 512+192(%rdi)
+
+
+
vpshufb %zmm8,%zmm4,%zmm4
vpxord %zmm10,%zmm4,%zmm4
vpshufb %zmm8,%zmm5,%zmm5
@@ -1054,28 +1049,21 @@
vaesenc %zmm23,%zmm2,%zmm2
vaesenc %zmm23,%zmm3,%zmm3
+
vextracti32x4 $1,%zmm10,%xmm4
vextracti32x4 $2,%zmm10,%xmm5
vextracti32x4 $3,%zmm10,%xmm6
vpxord %xmm4,%xmm10,%xmm10
vpternlogd $0x96,%xmm5,%xmm6,%xmm10
-
-
-
vpxord 0(%rdi),%zmm14,%zmm4
vpxord 64(%rdi),%zmm14,%zmm5
vpxord 128(%rdi),%zmm14,%zmm6
vpxord 192(%rdi),%zmm14,%zmm7
-
-
-
vaesenclast %zmm4,%zmm0,%zmm4
vaesenclast %zmm5,%zmm1,%zmm5
vaesenclast %zmm6,%zmm2,%zmm6
vaesenclast %zmm7,%zmm3,%zmm7
-
-
vmovdqu8 %zmm4,0(%rsi)
vmovdqu8 %zmm5,64(%rsi)
vmovdqu8 %zmm6,128(%rsi)
diff --git a/gen/bcm/aes-gcm-avx10-x86_64-win.asm b/gen/bcm/aes-gcm-avx10-x86_64-win.asm
index 733ae72..051a530 100644
--- a/gen/bcm/aes-gcm-avx10-x86_64-win.asm
+++ b/gen/bcm/aes-gcm-avx10-x86_64-win.asm
@@ -495,22 +495,14 @@
add rax,16
cmp r11,rax
jne NEAR $L$vaesenc_loop_first_4_vecs__func1
-
-
-
vpxord zmm4,zmm14,ZMMWORD[rcx]
vpxord zmm5,zmm14,ZMMWORD[64+rcx]
vpxord zmm6,zmm14,ZMMWORD[128+rcx]
vpxord zmm7,zmm14,ZMMWORD[192+rcx]
-
-
-
vaesenclast zmm4,zmm0,zmm4
vaesenclast zmm5,zmm1,zmm5
vaesenclast zmm6,zmm2,zmm6
vaesenclast zmm7,zmm3,zmm7
-
-
vmovdqu8 ZMMWORD[rdx],zmm4
vmovdqu8 ZMMWORD[64+rdx],zmm5
vmovdqu8 ZMMWORD[128+rdx],zmm6
@@ -521,6 +513,7 @@
add r8,-4*64
cmp r8,4*64-1
jbe NEAR $L$ghash_last_ciphertext_4x__func1
+
vbroadcasti32x4 zmm15,ZMMWORD[((-144))+r11]
vbroadcasti32x4 zmm16,ZMMWORD[((-128))+r11]
vbroadcasti32x4 zmm17,ZMMWORD[((-112))+r11]
@@ -530,6 +523,7 @@
vbroadcasti32x4 zmm21,ZMMWORD[((-48))+r11]
vbroadcasti32x4 zmm22,ZMMWORD[((-32))+r11]
vbroadcasti32x4 zmm23,ZMMWORD[((-16))+r11]
+
$L$crypt_loop_4x__func1:
@@ -583,6 +577,9 @@
prefetcht0 [((512+64))+rcx]
prefetcht0 [((512+128))+rcx]
prefetcht0 [((512+192))+rcx]
+
+
+
vpshufb zmm4,zmm4,zmm8
vpxord zmm4,zmm4,zmm10
vpshufb zmm5,zmm5,zmm8
@@ -672,28 +669,21 @@
vaesenc zmm2,zmm2,zmm23
vaesenc zmm3,zmm3,zmm23
+
vextracti32x4 xmm4,zmm10,1
vextracti32x4 xmm5,zmm10,2
vextracti32x4 xmm6,zmm10,3
vpxord xmm10,xmm10,xmm4
vpternlogd xmm10,xmm6,xmm5,0x96
-
-
-
vpxord zmm4,zmm14,ZMMWORD[rcx]
vpxord zmm5,zmm14,ZMMWORD[64+rcx]
vpxord zmm6,zmm14,ZMMWORD[128+rcx]
vpxord zmm7,zmm14,ZMMWORD[192+rcx]
-
-
-
vaesenclast zmm4,zmm0,zmm4
vaesenclast zmm5,zmm1,zmm5
vaesenclast zmm6,zmm2,zmm6
vaesenclast zmm7,zmm3,zmm7
-
-
vmovdqu8 ZMMWORD[rdx],zmm4
vmovdqu8 ZMMWORD[64+rdx],zmm5
vmovdqu8 ZMMWORD[128+rdx],zmm6
@@ -1003,6 +993,7 @@
vmovdqu8 zmm28,ZMMWORD[((256-192))+rdi]
vmovdqu8 zmm29,ZMMWORD[((256-128))+rdi]
vmovdqu8 zmm30,ZMMWORD[((256-64))+rdi]
+
vbroadcasti32x4 zmm15,ZMMWORD[((-144))+r11]
vbroadcasti32x4 zmm16,ZMMWORD[((-128))+r11]
vbroadcasti32x4 zmm17,ZMMWORD[((-112))+r11]
@@ -1012,6 +1003,7 @@
vbroadcasti32x4 zmm21,ZMMWORD[((-48))+r11]
vbroadcasti32x4 zmm22,ZMMWORD[((-32))+r11]
vbroadcasti32x4 zmm23,ZMMWORD[((-16))+r11]
+
$L$crypt_loop_4x__func2:
vmovdqu8 zmm4,ZMMWORD[rcx]
vmovdqu8 zmm5,ZMMWORD[64+rcx]
@@ -1069,6 +1061,9 @@
prefetcht0 [((512+64))+rcx]
prefetcht0 [((512+128))+rcx]
prefetcht0 [((512+192))+rcx]
+
+
+
vpshufb zmm4,zmm4,zmm8
vpxord zmm4,zmm4,zmm10
vpshufb zmm5,zmm5,zmm8
@@ -1158,28 +1153,21 @@
vaesenc zmm2,zmm2,zmm23
vaesenc zmm3,zmm3,zmm23
+
vextracti32x4 xmm4,zmm10,1
vextracti32x4 xmm5,zmm10,2
vextracti32x4 xmm6,zmm10,3
vpxord xmm10,xmm10,xmm4
vpternlogd xmm10,xmm6,xmm5,0x96
-
-
-
vpxord zmm4,zmm14,ZMMWORD[rcx]
vpxord zmm5,zmm14,ZMMWORD[64+rcx]
vpxord zmm6,zmm14,ZMMWORD[128+rcx]
vpxord zmm7,zmm14,ZMMWORD[192+rcx]
-
-
-
vaesenclast zmm4,zmm0,zmm4
vaesenclast zmm5,zmm1,zmm5
vaesenclast zmm6,zmm2,zmm6
vaesenclast zmm7,zmm3,zmm7
-
-
vmovdqu8 ZMMWORD[rdx],zmm4
vmovdqu8 ZMMWORD[64+rdx],zmm5
vmovdqu8 ZMMWORD[128+rdx],zmm6
