Google Git
Sign in
boringssl / boringssl.git / 6f4159567deb1a3d7b15108556360f97a2b57124 / . / crypto / fipsmodule / modes / asm / aes-gcm-avx10-x86_64.pl
blob: 269ac801a25e83862316301256914ce81a344272 [file] [log] [blame]
#!/usr/bin/env perl
# Copyright 2024 The BoringSSL Authors
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#------------------------------------------------------------------------------
#
# VAES and VPCLMULQDQ optimized AES-GCM for x86_64
#
# This file is based on aes-gcm-avx10-x86_64.S from the Linux kernel
# (https://git.kernel.org/linus/b06affb1cb580e13). The following notable
# changes have been made:
#
# - Relicensed under BoringSSL's preferred license.
#
# - Converted from GNU assembler to "perlasm". This was necessary for
# compatibility with BoringSSL's Windows builds which use NASM instead of the
# GNU assembler. It was also necessary for compatibility with the 'delocate'
# tool used in BoringSSL's FIPS builds.
#
# - Added support for the Windows ABI.
#
# - Changed function prototypes to be compatible with what BoringSSL wants.
#
# - Removed the optimized finalization function, as BoringSSL doesn't want it.
#
# - Added a single-block GHASH multiplication function, as BoringSSL needs this.
#
# - Added optimization for large amounts of AAD.
#
#------------------------------------------------------------------------------
#
# This file implements AES-GCM (Galois/Counter Mode) for x86_64 CPUs that
# support VAES (vector AES), VPCLMULQDQ (vector carryless multiplication), and
# either AVX512 or AVX10. Some of the functions, notably the encryption and
# decryption update functions which are the most performance-critical, are
# provided in two variants generated from a macro: one using 256-bit vectors
# (suffix: vaes_avx10_256) and one using 512-bit vectors (vaes_avx10_512). The
# other, "shared" functions (vaes_avx10) use at most 256-bit vectors.
#
# The functions that use 512-bit vectors are intended for CPUs that support
# 512-bit vectors *and* where using them doesn't cause significant
# downclocking. They require the following CPU features:
#
# VAES && VPCLMULQDQ && BMI2 && ((AVX512BW && AVX512VL) || AVX10/512)
#
# The other functions require the following CPU features:
#
# VAES && VPCLMULQDQ && BMI2 && ((AVX512BW && AVX512VL) || AVX10/256)
#
# Note that we use "avx10" in the names of the functions as a shorthand to
# really mean "AVX10 or a certain set of AVX512 features". Due to Intel's
# introduction of AVX512 and then its replacement by AVX10, there doesn't seem
# to be a simple way to name things that makes sense on all CPUs.
#
# Note that the macros that support both 256-bit and 512-bit vectors could
# fairly easily be changed to support 128-bit too. However, this would *not*
# be sufficient to allow the code to run on CPUs without AVX512 or AVX10,
# because the code heavily uses several features of these extensions other than
# the vector length: the increase in the number of SIMD registers from 16 to
# 32, masking support, and new instructions such as vpternlogd (which can do a
# three-argument XOR). These features are very useful for AES-GCM.
$flavour = shift;
$output = shift;
if ( $flavour =~ /\./ ) { $output = $flavour; undef $flavour; }
if ( $flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/ ) {
$win64 = 1;
@argregs = ( "%rcx", "%rdx", "%r8", "%r9" );
}
else {
$win64 = 0;
@argregs = ( "%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9" );
}
$0 =~ m/(.*[\/\\])[^\/\\]+$/;
$dir = $1;
( $xlate = "${dir}x86_64-xlate.pl" and -f $xlate )
or ( $xlate = "${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate )
or die "can't locate x86_64-xlate.pl";
open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT = *OUT;
sub _begin_func {
my ( $funcname, $uses_seh ) = @_;
$g_cur_func_name = $funcname;
$g_cur_func_uses_seh = $uses_seh;
@g_cur_func_saved_gpregs = ();
@g_cur_func_saved_xmmregs = ();
return <<___;
.globl $funcname
.type $funcname,\@abi-omnipotent
.align 32
$funcname:
.cfi_startproc
@{[ $uses_seh ? ".seh_startproc" : "" ]}
_CET_ENDBR
___
}
# Push a list of general purpose registers onto the stack.
sub _save_gpregs {
my @gpregs = @_;
my $code = "";
die "_save_gpregs requires uses_seh" unless $g_cur_func_uses_seh;
die "_save_gpregs can only be called once per function"
if @g_cur_func_saved_gpregs;
die "Order must be _save_gpregs, then _save_xmmregs"
if @g_cur_func_saved_xmmregs;
@g_cur_func_saved_gpregs = @gpregs;
for my $reg (@gpregs) {
$code .= "push $reg\n";
if ($win64) {
$code .= ".seh_pushreg $reg\n";
}
else {
$code .= ".cfi_push $reg\n";
}
}
return $code;
}
# Push a list of xmm registers onto the stack if the target is Windows.
sub _save_xmmregs {
my @xmmregs = @_;
my $num_xmmregs = scalar @xmmregs;
my $code = "";
die "_save_xmmregs requires uses_seh" unless $g_cur_func_uses_seh;
die "_save_xmmregs can only be called once per function"
if @g_cur_func_saved_xmmregs;
if ( $win64 and $num_xmmregs > 0 ) {
@g_cur_func_saved_xmmregs = @xmmregs;
my $is_misaligned = ( scalar @g_cur_func_saved_gpregs ) % 2 == 0;
my $alloc_size = 16 * $num_xmmregs + ( $is_misaligned ? 8 : 0 );
$code .= "sub \$$alloc_size, %rsp\n";
$code .= ".seh_stackalloc $alloc_size\n";
for my $i ( 0 .. $num_xmmregs - 1 ) {
my $reg_num = $xmmregs[$i];
my $pos = 16 * $i;
$code .= "movdqa %xmm$reg_num, $pos(%rsp)\n";
$code .= ".seh_savexmm %xmm$reg_num, $pos\n";
}
}
return $code;
}
sub _end_func {
my $code = "";
# Restore any xmm registers that were saved earlier.
my $num_xmmregs = scalar @g_cur_func_saved_xmmregs;
if ( $win64 and $num_xmmregs > 0 ) {
my $need_alignment = ( scalar @g_cur_func_saved_gpregs ) % 2 == 0;
my $alloc_size = 16 * $num_xmmregs + ( $need_alignment ? 8 : 0 );
for my $i ( 0 .. $num_xmmregs - 1 ) {
my $reg_num = $g_cur_func_saved_xmmregs[$i];
my $pos = 16 * $i;
$code .= "movdqa $pos(%rsp), %xmm$reg_num\n";
}
$code .= "add \$$alloc_size, %rsp\n";
}
# Restore any general purpose registers that were saved earlier.
for my $reg ( reverse @g_cur_func_saved_gpregs ) {
$code .= "pop $reg\n";
if ( !$win64 ) {
$code .= ".cfi_pop $reg\n";
}
}
$code .= <<___;
ret
@{[ $g_cur_func_uses_seh ? ".seh_endproc" : "" ]}
.cfi_endproc
.size $g_cur_func_name, . - $g_cur_func_name
___
return $code;
}
$code = <<___;
.section .rodata
.align 64
# A shuffle mask that reflects the bytes of 16-byte blocks
.Lbswap_mask:
.quad 0x08090a0b0c0d0e0f, 0x0001020304050607
# This is the GHASH reducing polynomial without its constant term, i.e.
# x^128 + x^7 + x^2 + x, represented using the backwards mapping
# between bits and polynomial coefficients.
#
# Alternatively, it can be interpreted as the naturally-ordered
# representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the
# "reversed" GHASH reducing polynomial without its x^128 term.
.Lgfpoly:
.quad 1, 0xc200000000000000
# Same as above, but with the (1 << 64) bit set.
.Lgfpoly_and_internal_carrybit:
.quad 1, 0xc200000000000001
# The below constants are used for incrementing the counter blocks.
# ctr_pattern points to the four 128-bit values [0, 1, 2, 3].
# inc_2blocks and inc_4blocks point to the single 128-bit values 2 and
# 4. Note that the same '2' is reused in ctr_pattern and inc_2blocks.
.Lctr_pattern:
.quad 0, 0
.quad 1, 0
.Linc_2blocks:
.quad 2, 0
.quad 3, 0
.Linc_4blocks:
.quad 4, 0
.text
___
# Number of powers of the hash key stored in the key struct. The powers are
# stored from highest (H^NUM_H_POWERS) to lowest (H^1).
$NUM_H_POWERS = 16;
$OFFSETOFEND_H_POWERS = $NUM_H_POWERS * 16;
# Offset to 'rounds' in AES_KEY struct
$OFFSETOF_AES_ROUNDS = 240;
# The current vector length in bytes
undef $VL;
# Set the vector length in bytes. This sets the VL variable and defines
# register aliases V0-V31 that map to the ymm or zmm registers.
sub _set_veclen {
($VL) = @_;
foreach my $i ( 0 .. 31 ) {
if ( $VL == 32 ) {
${"V${i}"} = "%ymm${i}";
}
elsif ( $VL == 64 ) {
${"V${i}"} = "%zmm${i}";
}
else {
die "Unsupported vector length";
}
}
}
# The _ghash_mul_step macro does one step of GHASH multiplication of the
# 128-bit lanes of \a by the corresponding 128-bit lanes of \b and storing the
# reduced products in \dst. \t0, \t1, and \t2 are temporary registers of the
# same size as \a and \b. To complete all steps, this must invoked with \i=0
# through \i=9. The division into steps allows users of this macro to
# optionally interleave the computation with other instructions. Users of this
# macro must preserve the parameter registers across steps.
#
# The multiplications are done in GHASH's representation of the finite field
# GF(2^128). Elements of GF(2^128) are represented as binary polynomials
# (i.e. polynomials whose coefficients are bits) modulo a reducing polynomial
# G. The GCM specification uses G = x^128 + x^7 + x^2 + x + 1. Addition is
# just XOR, while multiplication is more complex and has two parts: (a) do
# carryless multiplication of two 128-bit input polynomials to get a 256-bit
# intermediate product polynomial, and (b) reduce the intermediate product to
# 128 bits by adding multiples of G that cancel out terms in it. (Adding
# multiples of G doesn't change which field element the polynomial represents.)
#
# Unfortunately, the GCM specification maps bits to/from polynomial
# coefficients backwards from the natural order. In each byte it specifies the
# highest bit to be the lowest order polynomial coefficient, *not* the highest!
# This makes it nontrivial to work with the GHASH polynomials. We could
# reflect the bits, but x86 doesn't have an instruction that does that.
#
# Instead, we operate on the values without bit-reflecting them. This *mostly*
# just works, since XOR and carryless multiplication are symmetric with respect
# to bit order, but it has some consequences. First, due to GHASH's byte
# order, by skipping bit reflection, *byte* reflection becomes necessary to
# give the polynomial terms a consistent order. E.g., considering an N-bit
# value interpreted using the G = x^128 + x^7 + x^2 + x + 1 convention, bits 0
# through N-1 of the byte-reflected value represent the coefficients of x^(N-1)
# through x^0, whereas bits 0 through N-1 of the non-byte-reflected value
# represent x^7...x^0, x^15...x^8, ..., x^(N-1)...x^(N-8) which can't be worked
# with. Fortunately, x86's vpshufb instruction can do byte reflection.
#
# Second, forgoing the bit reflection causes an extra multiple of x (still
# using the G = x^128 + x^7 + x^2 + x + 1 convention) to be introduced by each
# multiplication. This is because an M-bit by N-bit carryless multiplication
# really produces a (M+N-1)-bit product, but in practice it's zero-extended to
# M+N bits. In the G = x^128 + x^7 + x^2 + x + 1 convention, which maps bits
# to polynomial coefficients backwards, this zero-extension actually changes
# the product by introducing an extra factor of x. Therefore, users of this
# macro must ensure that one of the inputs has an extra factor of x^-1, i.e.
# the multiplicative inverse of x, to cancel out the extra x.
#
# Third, the backwards coefficients convention is just confusing to work with,
# since it makes "low" and "high" in the polynomial math mean the opposite of
# their normal meaning in computer programming. This can be solved by using an
# alternative interpretation: the polynomial coefficients are understood to be
# in the natural order, and the multiplication is actually \a * \b * x^-128 mod
# x^128 + x^127 + x^126 + x^121 + 1. This doesn't change the inputs, outputs,
# or the implementation at all; it just changes the mathematical interpretation
# of what each instruction is doing. Starting from here, we'll use this
# alternative interpretation, as it's easier to understand the code that way.
#
# Moving onto the implementation, the vpclmulqdq instruction does 64 x 64 =>
# 128-bit carryless multiplication, so we break the 128 x 128 multiplication
# into parts as follows (the _L and _H suffixes denote low and high 64 bits):
#
# LO = a_L * b_L
# MI = (a_L * b_H) + (a_H * b_L)
# HI = a_H * b_H
#
# The 256-bit product is x^128*HI + x^64*MI + LO. LO, MI, and HI are 128-bit.
# Note that MI "overlaps" with LO and HI. We don't consolidate MI into LO and
# HI right away, since the way the reduction works makes that unnecessary.
#
# For the reduction, we cancel out the low 128 bits by adding multiples of G =
# x^128 + x^127 + x^126 + x^121 + 1. This is done by two iterations, each of
# which cancels out the next lowest 64 bits. Consider a value x^64*A + B,
# where A and B are 128-bit. Adding B_L*G to that value gives:
#
# x^64*A + B + B_L*G
# = x^64*A + x^64*B_H + B_L + B_L*(x^128 + x^127 + x^126 + x^121 + 1)
# = x^64*A + x^64*B_H + B_L + x^128*B_L + x^64*B_L*(x^63 + x^62 + x^57) + B_L
# = x^64*A + x^64*B_H + x^128*B_L + x^64*B_L*(x^63 + x^62 + x^57) + B_L + B_L
# = x^64*(A + B_H + x^64*B_L + B_L*(x^63 + x^62 + x^57))
#
# So: if we sum A, B with its halves swapped, and the low half of B times x^63
# + x^62 + x^57, we get a 128-bit value C where x^64*C is congruent to the
# original value x^64*A + B. I.e., the low 64 bits got canceled out.
#
# We just need to apply this twice: first to fold LO into MI, and second to
# fold the updated MI into HI.
#
# The needed three-argument XORs are done using the vpternlogd instruction with
# immediate 0x96, since this is faster than two vpxord instructions.
#
# A potential optimization, assuming that b is fixed per-key (if a is fixed
# per-key it would work the other way around), is to use one iteration of the
# reduction described above to precompute a value c such that x^64*c = b mod G,
# and then multiply a_L by c (and implicitly by x^64) instead of by b:
#
# MI = (a_L * c_L) + (a_H * b_L)
# HI = (a_L * c_H) + (a_H * b_H)
#
# This would eliminate the LO part of the intermediate product, which would
# eliminate the need to fold LO into MI. This would save two instructions,
# including a vpclmulqdq. However, we currently don't use this optimization
# because it would require twice as many per-key precomputed values.
#
# Using Karatsuba multiplication instead of "schoolbook" multiplication
# similarly would save a vpclmulqdq but does not seem to be worth it.
sub _ghash_mul_step {
my ( $i, $a, $b, $dst, $gfpoly, $t0, $t1, $t2 ) = @_;
if ( $i == 0 ) {
return "vpclmulqdq \$0x00, $a, $b, $t0\n" . # LO = a_L * b_L
"vpclmulqdq \$0x01, $a, $b, $t1\n"; # MI_0 = a_L * b_H
}
elsif ( $i == 1 ) {
return "vpclmulqdq \$0x10, $a, $b, $t2\n"; # MI_1 = a_H * b_L
}
elsif ( $i == 2 ) {
return "vpxord $t2, $t1, $t1\n"; # MI = MI_0 + MI_1
}
elsif ( $i == 3 ) {
return
"vpclmulqdq \$0x01, $t0, $gfpoly, $t2\n"; # LO_L*(x^63 + x^62 + x^57)
}
elsif ( $i == 4 ) {
return "vpshufd \$0x4e, $t0, $t0\n"; # Swap halves of LO
}
elsif ( $i == 5 ) {
return "vpternlogd \$0x96, $t2, $t0, $t1\n"; # Fold LO into MI
}
elsif ( $i == 6 ) {
return "vpclmulqdq \$0x11, $a, $b, $dst\n"; # HI = a_H * b_H
}
elsif ( $i == 7 ) {
return
"vpclmulqdq \$0x01, $t1, $gfpoly, $t0\n"; # MI_L*(x^63 + x^62 + x^57)
}
elsif ( $i == 8 ) {
return "vpshufd \$0x4e, $t1, $t1\n"; # Swap halves of MI
}
elsif ( $i == 9 ) {
return "vpternlogd \$0x96, $t0, $t1, $dst\n"; # Fold MI into HI
}
}
# GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and store
# the reduced products in \dst. See _ghash_mul_step for full explanation.
sub _ghash_mul {
my ( $a, $b, $dst, $gfpoly, $t0, $t1, $t2 ) = @_;
my $code = "";
for my $i ( 0 .. 9 ) {
$code .= _ghash_mul_step $i, $a, $b, $dst, $gfpoly, $t0, $t1, $t2;
}
return $code;
}
# GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and add the
# *unreduced* products to \lo, \mi, and \hi.
sub _ghash_mul_noreduce {
my ( $a, $b, $lo, $mi, $hi, $t0, $t1, $t2, $t3 ) = @_;
return <<___;
vpclmulqdq \$0x00, $a, $b, $t0 # a_L * b_L
vpclmulqdq \$0x01, $a, $b, $t1 # a_L * b_H
vpclmulqdq \$0x10, $a, $b, $t2 # a_H * b_L
vpclmulqdq \$0x11, $a, $b, $t3 # a_H * b_H
vpxord $t0, $lo, $lo
vpternlogd \$0x96, $t2, $t1, $mi
vpxord $t3, $hi, $hi
___
}
# Reduce the unreduced products from \lo, \mi, and \hi and store the 128-bit
# reduced products in \hi. See _ghash_mul_step for explanation of reduction.
sub _ghash_reduce {
my ( $lo, $mi, $hi, $gfpoly, $t0 ) = @_;
return <<___;
vpclmulqdq \$0x01, $lo, $gfpoly, $t0
vpshufd \$0x4e, $lo, $lo
vpternlogd \$0x96, $t0, $lo, $mi
vpclmulqdq \$0x01, $mi, $gfpoly, $t0
vpshufd \$0x4e, $mi, $mi
vpternlogd \$0x96, $t0, $mi, $hi
___
}
$g_init_macro_expansion_count = 0;
# void gcm_init_##suffix(u128 Htable[16], const uint64_t H[2]);
#
# Initialize |Htable| with powers of the GHASH subkey |H|.
#
# The powers are stored in the order H^NUM_H_POWERS to H^1.
#
# This macro supports both VL=32 and VL=64. _set_veclen must have been invoked
# with the desired length. In the VL=32 case, the function computes twice as
# many key powers than are actually used by the VL=32 GCM update functions.
# This is done to keep the key format the same regardless of vector length.
sub _aes_gcm_init {
my $local_label_suffix = "__func" . ++$g_init_macro_expansion_count;
# Function arguments
my ( $HTABLE, $H_PTR ) = @argregs[ 0 .. 1 ];
# Additional local variables. V0-V2 and %rax are used as temporaries.
my $POWERS_PTR = "%r8";
my $RNDKEYLAST_PTR = "%r9";
my ( $H_CUR, $H_CUR_YMM, $H_CUR_XMM ) = ( "$V3", "%ymm3", "%xmm3" );
my ( $H_INC, $H_INC_YMM, $H_INC_XMM ) = ( "$V4", "%ymm4", "%xmm4" );
my ( $GFPOLY, $GFPOLY_YMM, $GFPOLY_XMM ) = ( "$V5", "%ymm5", "%xmm5" );
my $code = <<___;
# Get pointer to lowest set of key powers (located at end of array).
lea $OFFSETOFEND_H_POWERS-$VL($HTABLE), $POWERS_PTR
# Load the byte-reflected hash subkey. BoringSSL provides it in
# byte-reflected form except the two halves are in the wrong order.
vpshufd \$0x4e, ($H_PTR), $H_CUR_XMM
# Finish preprocessing the first key power, H^1. Since this GHASH
# implementation operates directly on values with the backwards bit
# order specified by the GCM standard, it's necessary to preprocess the
# raw key as follows. First, reflect its bytes. Second, multiply it
# by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards
# interpretation of polynomial coefficients), which can also be
# interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121
# + 1 using the alternative, natural interpretation of polynomial
# coefficients. For details, see the comment above _ghash_mul_step.
#
# Either way, for the multiplication the concrete operation performed
# is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2
# << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit
# wide shift instruction, so instead double each of the two 64-bit
# halves and incorporate the internal carry bit into the value XOR'd.
vpshufd \$0xd3, $H_CUR_XMM, %xmm0
vpsrad \$31, %xmm0, %xmm0
vpaddq $H_CUR_XMM, $H_CUR_XMM, $H_CUR_XMM
# H_CUR_XMM ^= xmm0 & gfpoly_and_internal_carrybit
vpternlogd \$0x78, .Lgfpoly_and_internal_carrybit(%rip), %xmm0, $H_CUR_XMM
# Load the gfpoly constant.
vbroadcasti32x4 .Lgfpoly(%rip), $GFPOLY
# Square H^1 to get H^2.
#
# Note that as with H^1, all higher key powers also need an extra
# factor of x^-1 (or x using the natural interpretation). Nothing
# special needs to be done to make this happen, though: H^1 * H^1 would
# end up with two factors of x^-1, but the multiplication consumes one.
# So the product H^2 ends up with the desired one factor of x^-1.
@{[ _ghash_mul $H_CUR_XMM, $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM,
"%xmm0", "%xmm1", "%xmm2" ]}
# Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2].
vinserti128 \$1, $H_CUR_XMM, $H_INC_YMM, $H_CUR_YMM
vinserti128 \$1, $H_INC_XMM, $H_INC_YMM, $H_INC_YMM
___
if ( $VL == 64 ) {
# Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4].
$code .= <<___;
@{[ _ghash_mul $H_INC_YMM, $H_CUR_YMM, $H_INC_YMM, $GFPOLY_YMM,
"%ymm0", "%ymm1", "%ymm2" ]}
vinserti64x4 \$1, $H_CUR_YMM, $H_INC, $H_CUR
vshufi64x2 \$0, $H_INC, $H_INC, $H_INC
___
}
$code .= <<___;
# Store the lowest set of key powers.
vmovdqu8 $H_CUR, ($POWERS_PTR)
# Compute and store the remaining key powers. With VL=32, repeatedly
# multiply [H^(i+1), H^i] by [H^2, H^2] to get [H^(i+3), H^(i+2)].
# With VL=64, repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by
# [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)].
mov \$@{[ $NUM_H_POWERS*16/$VL - 1 ]}, %eax
.Lprecompute_next$local_label_suffix:
sub \$$VL, $POWERS_PTR
@{[ _ghash_mul $H_INC, $H_CUR, $H_CUR, $GFPOLY, $V0, $V1, $V2 ]}
vmovdqu8 $H_CUR, ($POWERS_PTR)
dec %eax
jnz .Lprecompute_next$local_label_suffix
vzeroupper # This is needed after using ymm or zmm registers.
___
return $code;
}
# XOR together the 128-bit lanes of \src (whose low lane is \src_xmm) and store
# the result in \dst_xmm. This implicitly zeroizes the other lanes of dst.
sub _horizontal_xor {
my ( $src, $src_xmm, $dst_xmm, $t0_xmm, $t1_xmm, $t2_xmm ) = @_;
if ( $VL == 32 ) {
return <<___;
vextracti32x4 \$1, $src, $t0_xmm
vpxord $t0_xmm, $src_xmm, $dst_xmm
___
}
elsif ( $VL == 64 ) {
return <<___;
vextracti32x4 \$1, $src, $t0_xmm
vextracti32x4 \$2, $src, $t1_xmm
vextracti32x4 \$3, $src, $t2_xmm
vpxord $t0_xmm, $src_xmm, $dst_xmm
vpternlogd \$0x96, $t1_xmm, $t2_xmm, $dst_xmm
___
}
else {
die "Unsupported vector length";
}
}
# Do one step of the GHASH update of the data blocks given in the vector
# registers GHASHDATA[0-3]. \i specifies the step to do, 0 through 9. The
# division into steps allows users of this macro to optionally interleave the
# computation with other instructions. This macro uses the vector register
# GHASH_ACC as input/output; GHASHDATA[0-3] as inputs that are clobbered;
# H_POW[4-1], GFPOLY, and BSWAP_MASK as inputs that aren't clobbered; and
# GHASHTMP[0-2] as temporaries. This macro handles the byte-reflection of the
# data blocks. The parameter registers must be preserved across steps.
#
# The GHASH update does: GHASH_ACC = H_POW4*(GHASHDATA0 + GHASH_ACC) +
# H_POW3*GHASHDATA1 + H_POW2*GHASHDATA2 + H_POW1*GHASHDATA3, where the
# operations are vectorized operations on vectors of 16-byte blocks. E.g.,
# with VL=32 there are 2 blocks per vector and the vectorized terms correspond
# to the following non-vectorized terms:
#
# H_POW4*(GHASHDATA0 + GHASH_ACC) => H^8*(blk0 + GHASH_ACC_XMM) and H^7*(blk1 + 0)
# H_POW3*GHASHDATA1 => H^6*blk2 and H^5*blk3
# H_POW2*GHASHDATA2 => H^4*blk4 and H^3*blk5
# H_POW1*GHASHDATA3 => H^2*blk6 and H^1*blk7
#
# With VL=64, we use 4 blocks/vector, H^16 through H^1, and blk0 through blk15.
#
# More concretely, this code does:
# - Do vectorized "schoolbook" multiplications to compute the intermediate
# 256-bit product of each block and its corresponding hash key power.
# There are 4*VL/16 of these intermediate products.
# - Sum (XOR) the intermediate 256-bit products across vectors. This leaves
# VL/16 256-bit intermediate values.
# - Do a vectorized reduction of these 256-bit intermediate values to
# 128-bits each. This leaves VL/16 128-bit intermediate values.
# - Sum (XOR) these values and store the 128-bit result in GHASH_ACC_XMM.
#
# See _ghash_mul_step for the full explanation of the operations performed for
# each individual finite field multiplication and reduction.
sub _ghash_step_4x {
my ($i) = @_;
if ( $i == 0 ) {
return <<___;
vpshufb $BSWAP_MASK, $GHASHDATA0, $GHASHDATA0
vpxord $GHASH_ACC, $GHASHDATA0, $GHASHDATA0
vpshufb $BSWAP_MASK, $GHASHDATA1, $GHASHDATA1
vpshufb $BSWAP_MASK, $GHASHDATA2, $GHASHDATA2
___
}
elsif ( $i == 1 ) {
return <<___;
vpshufb $BSWAP_MASK, $GHASHDATA3, $GHASHDATA3
vpclmulqdq \$0x00, $H_POW4, $GHASHDATA0, $GHASH_ACC # LO_0
vpclmulqdq \$0x00, $H_POW3, $GHASHDATA1, $GHASHTMP0 # LO_1
vpclmulqdq \$0x00, $H_POW2, $GHASHDATA2, $GHASHTMP1 # LO_2
___
}
elsif ( $i == 2 ) {
return <<___;
vpxord $GHASHTMP0, $GHASH_ACC, $GHASH_ACC # sum(LO_{1,0})
vpclmulqdq \$0x00, $H_POW1, $GHASHDATA3, $GHASHTMP2 # LO_3
vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASH_ACC # LO = sum(LO_{3,2,1,0})
vpclmulqdq \$0x01, $H_POW4, $GHASHDATA0, $GHASHTMP0 # MI_0
___
}
elsif ( $i == 3 ) {
return <<___;
vpclmulqdq \$0x01, $H_POW3, $GHASHDATA1, $GHASHTMP1 # MI_1
vpclmulqdq \$0x01, $H_POW2, $GHASHDATA2, $GHASHTMP2 # MI_2
vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASHTMP0 # sum(MI_{2,1,0})
vpclmulqdq \$0x01, $H_POW1, $GHASHDATA3, $GHASHTMP1 # MI_3
___
}
elsif ( $i == 4 ) {
return <<___;
vpclmulqdq \$0x10, $H_POW4, $GHASHDATA0, $GHASHTMP2 # MI_4
vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASHTMP0 # sum(MI_{4,3,2,1,0})
vpclmulqdq \$0x10, $H_POW3, $GHASHDATA1, $GHASHTMP1 # MI_5
vpclmulqdq \$0x10, $H_POW2, $GHASHDATA2, $GHASHTMP2 # MI_6
___
}
elsif ( $i == 5 ) {
return <<___;
vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASHTMP0 # sum(MI_{6,5,4,3,2,1,0})
vpclmulqdq \$0x01, $GHASH_ACC, $GFPOLY, $GHASHTMP2 # LO_L*(x^63 + x^62 + x^57)
vpclmulqdq \$0x10, $H_POW1, $GHASHDATA3, $GHASHTMP1 # MI_7
vpxord $GHASHTMP1, $GHASHTMP0, $GHASHTMP0 # MI = sum(MI_{7,6,5,4,3,2,1,0})
___
}
elsif ( $i == 6 ) {
return <<___;
vpshufd \$0x4e, $GHASH_ACC, $GHASH_ACC # Swap halves of LO
vpclmulqdq \$0x11, $H_POW4, $GHASHDATA0, $GHASHDATA0 # HI_0
vpclmulqdq \$0x11, $H_POW3, $GHASHDATA1, $GHASHDATA1 # HI_1
vpclmulqdq \$0x11, $H_POW2, $GHASHDATA2, $GHASHDATA2 # HI_2
___
}
elsif ( $i == 7 ) {
return <<___;
vpternlogd \$0x96, $GHASHTMP2, $GHASH_ACC, $GHASHTMP0 # Fold LO into MI
vpclmulqdq \$0x11, $H_POW1, $GHASHDATA3, $GHASHDATA3 # HI_3
vpternlogd \$0x96, $GHASHDATA2, $GHASHDATA1, $GHASHDATA0 # sum(HI_{2,1,0})
vpclmulqdq \$0x01, $GHASHTMP0, $GFPOLY, $GHASHTMP1 # MI_L*(x^63 + x^62 + x^57)
___
}
elsif ( $i == 8 ) {
return <<___;
vpxord $GHASHDATA3, $GHASHDATA0, $GHASH_ACC # HI = sum(HI_{3,2,1,0})
vpshufd \$0x4e, $GHASHTMP0, $GHASHTMP0 # Swap halves of MI
vpternlogd \$0x96, $GHASHTMP1, $GHASHTMP0, $GHASH_ACC # Fold MI into HI
___
}
elsif ( $i == 9 ) {
return _horizontal_xor $GHASH_ACC, $GHASH_ACC_XMM, $GHASH_ACC_XMM,
$GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM;
}
}
# Update GHASH with the blocks given in GHASHDATA[0-3].
# See _ghash_step_4x for full explanation.
sub _ghash_4x {
my $code = "";
for my $i ( 0 .. 9 ) {
$code .= _ghash_step_4x $i;
}
return $code;
}
$g_ghash_macro_expansion_count = 0;
# void gcm_ghash_##suffix(uint8_t Xi[16], const u128 Htable[16],
# const uint8_t *in, size_t len);
#
# This macro generates the body of a GHASH update function with the above
# prototype. This macro supports both VL=32 and VL=64. _set_veclen must have
# been invoked with the desired length.
#
# The generated function processes the AAD (Additional Authenticated Data) in
# GCM. Using the key |Htable|, it updates the GHASH accumulator |Xi| with the
# data given by |in| and |len|. On the first call, |Xi| must be all zeroes.
# |len| must be a multiple of 16.
#
# This function handles large amounts of AAD efficiently, while also keeping the
# overhead low for small amounts of AAD which is the common case. TLS uses less
# than one block of AAD, but (uncommonly) other use cases may use much more.
sub _ghash_update {
my $local_label_suffix = "__func" . ++$g_ghash_macro_expansion_count;
my $code = "";
# Function arguments
my ( $GHASH_ACC_PTR, $H_POWERS, $AAD, $AADLEN ) = @argregs[ 0 .. 3 ];
# Additional local variables
( $GHASHDATA0, $GHASHDATA0_XMM ) = ( $V0, "%xmm0" );
( $GHASHDATA1, $GHASHDATA1_XMM ) = ( $V1, "%xmm1" );
( $GHASHDATA2, $GHASHDATA2_XMM ) = ( $V2, "%xmm2" );
( $GHASHDATA3, $GHASHDATA3_XMM ) = ( $V3, "%xmm3" );
( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( $V4, "%xmm4" );
( $GHASH_ACC, $GHASH_ACC_XMM ) = ( $V5, "%xmm5" );
( $H_POW4, $H_POW3, $H_POW2 ) = ( $V6, $V7, $V8 );
( $H_POW1, $H_POW1_XMM ) = ( $V9, "%xmm9" );
( $GFPOLY, $GFPOLY_XMM ) = ( $V10, "%xmm10" );
( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( $V11, $V12, $V13 );
$code .= <<___;
@{[ _save_xmmregs (6 .. 13) ]}
.seh_endprologue
# Load the bswap_mask and gfpoly constants. Since AADLEN is usually small,
# usually only 128-bit vectors will be used. So as an optimization, don't
# broadcast these constants to all 128-bit lanes quite yet.
vmovdqu .Lbswap_mask(%rip), $BSWAP_MASK_XMM
vmovdqu .Lgfpoly(%rip), $GFPOLY_XMM
# Load the GHASH accumulator.
vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC_XMM
vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
# Optimize for AADLEN < VL by checking for AADLEN < VL before AADLEN < 4*VL.
cmp \$$VL, $AADLEN
jb .Laad_blockbyblock$local_label_suffix
# AADLEN >= VL, so we'll operate on full vectors. Broadcast bswap_mask and
# gfpoly to all 128-bit lanes.
vshufi64x2 \$0, $BSWAP_MASK, $BSWAP_MASK, $BSWAP_MASK
vshufi64x2 \$0, $GFPOLY, $GFPOLY, $GFPOLY
# Load the lowest set of key powers.
vmovdqu8 $OFFSETOFEND_H_POWERS-1*$VL($H_POWERS), $H_POW1
cmp \$4*$VL-1, $AADLEN
jbe .Laad_loop_1x$local_label_suffix
# AADLEN >= 4*VL. Load the higher key powers.
vmovdqu8 $OFFSETOFEND_H_POWERS-4*$VL($H_POWERS), $H_POW4
vmovdqu8 $OFFSETOFEND_H_POWERS-3*$VL($H_POWERS), $H_POW3
vmovdqu8 $OFFSETOFEND_H_POWERS-2*$VL($H_POWERS), $H_POW2
# Update GHASH with 4*VL bytes of AAD at a time.
.Laad_loop_4x$local_label_suffix:
vmovdqu8 0*$VL($AAD), $GHASHDATA0
vmovdqu8 1*$VL($AAD), $GHASHDATA1
vmovdqu8 2*$VL($AAD), $GHASHDATA2
vmovdqu8 3*$VL($AAD), $GHASHDATA3
@{[ _ghash_4x ]}
sub \$-4*$VL, $AAD # shorter than 'add 4*VL' when VL=32
add \$-4*$VL, $AADLEN
cmp \$4*$VL-1, $AADLEN
ja .Laad_loop_4x$local_label_suffix
# Update GHASH with VL bytes of AAD at a time.
cmp \$$VL, $AADLEN
jb .Laad_large_done$local_label_suffix
.Laad_loop_1x$local_label_suffix:
vmovdqu8 ($AAD), $GHASHDATA0
vpshufb $BSWAP_MASK, $GHASHDATA0, $GHASHDATA0
vpxord $GHASHDATA0, $GHASH_ACC, $GHASH_ACC
@{[ _ghash_mul $H_POW1, $GHASH_ACC, $GHASH_ACC, $GFPOLY,
$GHASHDATA0, $GHASHDATA1, $GHASHDATA2 ]}
@{[ _horizontal_xor $GHASH_ACC, $GHASH_ACC_XMM, $GHASH_ACC_XMM,
$GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM ]}
add \$$VL, $AAD
sub \$$VL, $AADLEN
cmp \$$VL, $AADLEN
jae .Laad_loop_1x$local_label_suffix
.Laad_large_done$local_label_suffix:
# Issue the vzeroupper that is needed after using ymm or zmm registers.
# Do it here instead of at the end, to minimize overhead for small AADLEN.
vzeroupper
# GHASH the remaining data 16 bytes at a time, using xmm registers only.
.Laad_blockbyblock$local_label_suffix:
test $AADLEN, $AADLEN
jz .Laad_done$local_label_suffix
vmovdqu $OFFSETOFEND_H_POWERS-16($H_POWERS), $H_POW1_XMM
.Laad_loop_blockbyblock$local_label_suffix:
vmovdqu ($AAD), $GHASHDATA0_XMM
vpshufb $BSWAP_MASK_XMM, $GHASHDATA0_XMM, $GHASHDATA0_XMM
vpxor $GHASHDATA0_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
@{[ _ghash_mul $H_POW1_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GFPOLY_XMM,
$GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM ]}
add \$16, $AAD
sub \$16, $AADLEN
jnz .Laad_loop_blockbyblock$local_label_suffix
.Laad_done$local_label_suffix:
# Store the updated GHASH accumulator back to memory.
vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
vmovdqu $GHASH_ACC_XMM, ($GHASH_ACC_PTR)
___
return $code;
}
# Do one non-last round of AES encryption on the counter blocks in V0-V3 using
# the round key that has been broadcast to all 128-bit lanes of \round_key.
sub _vaesenc_4x {
my ($round_key) = @_;
return <<___;
vaesenc $round_key, $V0, $V0
vaesenc $round_key, $V1, $V1
vaesenc $round_key, $V2, $V2
vaesenc $round_key, $V3, $V3
___
}
# Start the AES encryption of four vectors of counter blocks.
sub _ctr_begin_4x {
return <<___;
# Increment LE_CTR four times to generate four vectors of little-endian
# counter blocks, swap each to big-endian, and store them in V0-V3.
vpshufb $BSWAP_MASK, $LE_CTR, $V0
vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
vpshufb $BSWAP_MASK, $LE_CTR, $V1
vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
vpshufb $BSWAP_MASK, $LE_CTR, $V2
vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
vpshufb $BSWAP_MASK, $LE_CTR, $V3
vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
# AES "round zero": XOR in the zero-th round key.
vpxord $RNDKEY0, $V0, $V0
vpxord $RNDKEY0, $V1, $V1
vpxord $RNDKEY0, $V2, $V2
vpxord $RNDKEY0, $V3, $V3
___
}
# Do the last AES round for four vectors of counter blocks V0-V3, XOR source
# data with the resulting keystream, and write the result to DST and
# GHASHDATA[0-3]. (Implementation differs slightly, but has the same effect.)
sub _aesenclast_and_xor_4x {
return <<___;
# XOR the source data with the last round key, saving the result in
# GHASHDATA[0-3]. This reduces latency by taking advantage of the
# property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
vpxord 0*$VL($SRC), $RNDKEYLAST, $GHASHDATA0
vpxord 1*$VL($SRC), $RNDKEYLAST, $GHASHDATA1
vpxord 2*$VL($SRC), $RNDKEYLAST, $GHASHDATA2
vpxord 3*$VL($SRC), $RNDKEYLAST, $GHASHDATA3
# Do the last AES round. This handles the XOR with the source data
# too, as per the optimization described above.
vaesenclast $GHASHDATA0, $V0, $GHASHDATA0
vaesenclast $GHASHDATA1, $V1, $GHASHDATA1
vaesenclast $GHASHDATA2, $V2, $GHASHDATA2
vaesenclast $GHASHDATA3, $V3, $GHASHDATA3
# Store the en/decrypted data to DST.
vmovdqu8 $GHASHDATA0, 0*$VL($DST)
vmovdqu8 $GHASHDATA1, 1*$VL($DST)
vmovdqu8 $GHASHDATA2, 2*$VL($DST)
vmovdqu8 $GHASHDATA3, 3*$VL($DST)
___
}
$g_update_macro_expansion_count = 0;
# void aes_gcm_{enc,dec}_update_##suffix(const uint8_t *in, uint8_t *out,
# size_t len, const AES_KEY *key,
# const uint8_t ivec[16],
# const u128 Htable[16],
# uint8_t Xi[16]);
#
# This macro generates a GCM encryption or decryption update function with the
# above prototype (with \enc selecting which one). This macro supports both
# VL=32 and VL=64. _set_veclen must have been invoked with the desired length.
#
# This function computes the next portion of the CTR keystream, XOR's it with
# |len| bytes from |in|, and writes the resulting encrypted or decrypted data
# to |out|. It also updates the GHASH accumulator |Xi| using the next |len|
# ciphertext bytes.
#
# |len| must be a multiple of 16, except on the last call where it can be any
# length. The caller must do any buffering needed to ensure this. Both
# in-place and out-of-place en/decryption are supported.
#
# |ivec| must give the current counter in big-endian format. This function
# loads the counter from |ivec| and increments the loaded counter as needed, but
# it does *not* store the updated counter back to |ivec|. The caller must
# update |ivec| if any more data segments follow. Internally, only the low
# 32-bit word of the counter is incremented, following the GCM standard.
sub _aes_gcm_update {
my $local_label_suffix = "__func" . ++$g_update_macro_expansion_count;
my ($enc) = @_;
my $code = "";
# Function arguments
( $SRC, $DST, $DATALEN, $AESKEY, $BE_CTR_PTR, $H_POWERS, $GHASH_ACC_PTR ) =
$win64
? ( @argregs[ 0 .. 3 ], "%rsi", "%rdi", "%r12" )
: ( @argregs[ 0 .. 5 ], "%r12" );
# Additional local variables
# %rax, %k1, and %k2 are used as temporary registers. BE_CTR_PTR is
# also available as a temporary register after the counter is loaded.
# AES key length in bytes
( $AESKEYLEN, $AESKEYLEN64 ) = ( "%r10d", "%r10" );
# Pointer to the last AES round key for the chosen AES variant
$RNDKEYLAST_PTR = "%r11";
# In the main loop, V0-V3 are used as AES input and output. Elsewhere
# they are used as temporary registers.
# GHASHDATA[0-3] hold the ciphertext blocks and GHASH input data.
( $GHASHDATA0, $GHASHDATA0_XMM ) = ( $V4, "%xmm4" );
( $GHASHDATA1, $GHASHDATA1_XMM ) = ( $V5, "%xmm5" );
( $GHASHDATA2, $GHASHDATA2_XMM ) = ( $V6, "%xmm6" );
( $GHASHDATA3, $GHASHDATA3_XMM ) = ( $V7, "%xmm7" );
# BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values
# using vpshufb, copied to all 128-bit lanes.
( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( $V8, "%xmm8" );
# RNDKEY temporarily holds the next AES round key.
$RNDKEY = $V9;
# GHASH_ACC is the accumulator variable for GHASH. When fully reduced,
# only the lowest 128-bit lane can be nonzero. When not fully reduced,
# more than one lane may be used, and they need to be XOR'd together.
( $GHASH_ACC, $GHASH_ACC_XMM ) = ( $V10, "%xmm10" );
# LE_CTR_INC is the vector of 32-bit words that need to be added to a
# vector of little-endian counter blocks to advance it forwards.
$LE_CTR_INC = $V11;
# LE_CTR contains the next set of little-endian counter blocks.
$LE_CTR = $V12;
# RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys,
# copied to all 128-bit lanes. RNDKEY0 is the zero-th round key,
# RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last.
(
$RNDKEY0, $RNDKEYLAST, $RNDKEY_M9, $RNDKEY_M8,
$RNDKEY_M7, $RNDKEY_M6, $RNDKEY_M5, $RNDKEY_M4,
$RNDKEY_M3, $RNDKEY_M2, $RNDKEY_M1
) = ( $V13, $V14, $V15, $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23 );
# GHASHTMP[0-2] are temporary variables used by _ghash_step_4x. These
# cannot coincide with anything used for AES encryption, since for
# performance reasons GHASH and AES encryption are interleaved.
( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( $V24, $V25, $V26 );
# H_POW[4-1] contain the powers of the hash key H^(4*VL/16)...H^1. The
# descending numbering reflects the order of the key powers.
( $H_POW4, $H_POW3, $H_POW2, $H_POW1 ) = ( $V27, $V28, $V29, $V30 );
# GFPOLY contains the .Lgfpoly constant, copied to all 128-bit lanes.
$GFPOLY = $V31;
if ($win64) {
$code .= <<___;
@{[ _save_gpregs $BE_CTR_PTR, $H_POWERS, $GHASH_ACC_PTR ]}
mov 64(%rsp), $BE_CTR_PTR # arg5
mov 72(%rsp), $H_POWERS # arg6
mov 80(%rsp), $GHASH_ACC_PTR # arg7
@{[ _save_xmmregs (6 .. 15) ]}
.seh_endprologue
___
}
else {
$code .= <<___;
@{[ _save_gpregs $GHASH_ACC_PTR ]}
mov 16(%rsp), $GHASH_ACC_PTR # arg7
___
}
if ($enc) {
$code .= <<___;
#ifdef BORINGSSL_DISPATCH_TEST
.extern BORINGSSL_function_hit
movb \$1,BORINGSSL_function_hit+@{[ $VL < 64 ? 6 : 7 ]}(%rip)
#endif
___
}
$code .= <<___;
# Load some constants.
vbroadcasti32x4 .Lbswap_mask(%rip), $BSWAP_MASK
vbroadcasti32x4 .Lgfpoly(%rip), $GFPOLY
# Load the GHASH accumulator and the starting counter.
# BoringSSL passes these values in big endian format.
vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC_XMM
vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
vbroadcasti32x4 ($BE_CTR_PTR), $LE_CTR
vpshufb $BSWAP_MASK, $LE_CTR, $LE_CTR
# Load the AES key length in bytes. BoringSSL stores number of rounds
# minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20.
movl $OFFSETOF_AES_ROUNDS($AESKEY), $AESKEYLEN
lea -20(,$AESKEYLEN,4), $AESKEYLEN
# Make RNDKEYLAST_PTR point to the last AES round key. This is the
# round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
# respectively. Then load the zero-th and last round keys.
lea 6*16($AESKEY,$AESKEYLEN64,4), $RNDKEYLAST_PTR
vbroadcasti32x4 ($AESKEY), $RNDKEY0
vbroadcasti32x4 ($RNDKEYLAST_PTR), $RNDKEYLAST
# Finish initializing LE_CTR by adding [0, 1, ...] to its low words.
vpaddd .Lctr_pattern(%rip), $LE_CTR, $LE_CTR
# Initialize LE_CTR_INC to contain VL/16 in all 128-bit lanes.
vbroadcasti32x4 .Linc_@{[ $VL / 16 ]}blocks(%rip), $LE_CTR_INC
# If there are at least 4*VL bytes of data, then continue into the loop
# that processes 4*VL bytes of data at a time. Otherwise skip it.
cmp \$4*$VL-1, $DATALEN
jbe .Lcrypt_loop_4x_done$local_label_suffix
# Load powers of the hash key.
vmovdqu8 $OFFSETOFEND_H_POWERS-4*$VL($H_POWERS), $H_POW4
vmovdqu8 $OFFSETOFEND_H_POWERS-3*$VL($H_POWERS), $H_POW3
vmovdqu8 $OFFSETOFEND_H_POWERS-2*$VL($H_POWERS), $H_POW2
vmovdqu8 $OFFSETOFEND_H_POWERS-1*$VL($H_POWERS), $H_POW1
___
# Main loop: en/decrypt and hash 4 vectors at a time.
#
# When possible, interleave the AES encryption of the counter blocks
# with the GHASH update of the ciphertext blocks. This improves
# performance on many CPUs because the execution ports used by the VAES
# instructions often differ from those used by vpclmulqdq and other
# instructions used in GHASH. For example, many Intel CPUs dispatch
# vaesenc to ports 0 and 1 and vpclmulqdq to port 5.
#
# The interleaving is easiest to do during decryption, since during
# decryption the ciphertext blocks are immediately available. For
# encryption, instead encrypt the first set of blocks, then hash those
# blocks while encrypting the next set of blocks, repeat that as
# needed, and finally hash the last set of blocks.
if ($enc) {
$code .= <<___;
# Encrypt the first 4 vectors of plaintext blocks. Leave the resulting
# ciphertext in GHASHDATA[0-3] for GHASH.
@{[ _ctr_begin_4x ]}
lea 16($AESKEY), %rax
.Lvaesenc_loop_first_4_vecs$local_label_suffix:
vbroadcasti32x4 (%rax), $RNDKEY
@{[ _vaesenc_4x $RNDKEY ]}
add \$16, %rax
cmp %rax, $RNDKEYLAST_PTR
jne .Lvaesenc_loop_first_4_vecs$local_label_suffix
@{[ _aesenclast_and_xor_4x ]}
sub \$-4*$VL, $SRC # shorter than 'add 4*VL' when VL=32
sub \$-4*$VL, $DST
add \$-4*$VL, $DATALEN
cmp \$4*$VL-1, $DATALEN
jbe .Lghash_last_ciphertext_4x$local_label_suffix
___
}
# Cache as many additional AES round keys as possible.
for my $i ( reverse 1 .. 9 ) {
$code .= <<___;
vbroadcasti32x4 -$i*16($RNDKEYLAST_PTR), ${"RNDKEY_M$i"}
___
}
$code .= <<___;
.Lcrypt_loop_4x$local_label_suffix:
___
# If decrypting, load more ciphertext blocks into GHASHDATA[0-3]. If
# encrypting, GHASHDATA[0-3] already contain the previous ciphertext.
if ( !$enc ) {
$code .= <<___;
vmovdqu8 0*$VL($SRC), $GHASHDATA0
vmovdqu8 1*$VL($SRC), $GHASHDATA1
vmovdqu8 2*$VL($SRC), $GHASHDATA2
vmovdqu8 3*$VL($SRC), $GHASHDATA3
___
}
$code .= <<___;
# Start the AES encryption of the counter blocks.
@{[ _ctr_begin_4x ]}
cmp \$24, $AESKEYLEN
jl .Laes128$local_label_suffix
je .Laes192$local_label_suffix
# AES-256
vbroadcasti32x4 -13*16($RNDKEYLAST_PTR), $RNDKEY
@{[ _vaesenc_4x $RNDKEY ]}
vbroadcasti32x4 -12*16($RNDKEYLAST_PTR), $RNDKEY
@{[ _vaesenc_4x $RNDKEY ]}
.Laes192$local_label_suffix:
vbroadcasti32x4 -11*16($RNDKEYLAST_PTR), $RNDKEY
@{[ _vaesenc_4x $RNDKEY ]}
vbroadcasti32x4 -10*16($RNDKEYLAST_PTR), $RNDKEY
@{[ _vaesenc_4x $RNDKEY ]}
.Laes128$local_label_suffix:
___
# Finish the AES encryption of the counter blocks in V0-V3, interleaved
# with the GHASH update of the ciphertext blocks in GHASHDATA[0-3].
for my $i ( reverse 1 .. 9 ) {
$code .= <<___;
@{[ _ghash_step_4x (9 - $i) ]}
@{[ _vaesenc_4x ${"RNDKEY_M$i"} ]}
___
}
$code .= <<___;
@{[ _ghash_step_4x 9 ]}
@{[ _aesenclast_and_xor_4x ]}
sub \$-4*$VL, $SRC # shorter than 'add 4*VL' when VL=32
sub \$-4*$VL, $DST
add \$-4*$VL, $DATALEN
cmp \$4*$VL-1, $DATALEN
ja .Lcrypt_loop_4x$local_label_suffix
___
if ($enc) {
# Update GHASH with the last set of ciphertext blocks.
$code .= <<___;
.Lghash_last_ciphertext_4x$local_label_suffix:
@{[ _ghash_4x ]}
___
}
my $POWERS_PTR = $BE_CTR_PTR; # BE_CTR_PTR is free to be reused.
$code .= <<___;
.Lcrypt_loop_4x_done$local_label_suffix:
# Check whether any data remains.
test $DATALEN, $DATALEN
jz .Ldone$local_label_suffix
# The data length isn't a multiple of 4*VL. Process the remaining data
# of length 1 <= DATALEN < 4*VL, up to one vector (VL bytes) at a time.
# Going one vector at a time may seem inefficient compared to having
# separate code paths for each possible number of vectors remaining.
# However, using a loop keeps the code size down, and it performs
# surprising well; modern CPUs will start executing the next iteration
# before the previous one finishes and also predict the number of loop
# iterations. For a similar reason, we roll up the AES rounds.
#
# On the last iteration, the remaining length may be less than VL.
# Handle this using masking.
#
# Since there are enough key powers available for all remaining data,
# there is no need to do a GHASH reduction after each iteration.
# Instead, multiply each remaining block by its own key power, and only
# do a GHASH reduction at the very end.
# Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N
# is the number of blocks that remain.
mov $DATALEN, %rax
neg %rax
and \$-16, %rax # -round_up(DATALEN, 16)
lea $OFFSETOFEND_H_POWERS($H_POWERS,%rax), $POWERS_PTR
___
# Start collecting the unreduced GHASH intermediate value LO, MI, HI.
my ( $LO, $LO_XMM ) = ( $GHASHDATA0, $GHASHDATA0_XMM );
my ( $MI, $MI_XMM ) = ( $GHASHDATA1, $GHASHDATA1_XMM );
my ( $HI, $HI_XMM ) = ( $GHASHDATA2, $GHASHDATA2_XMM );
$code .= <<___;
vpxor $LO_XMM, $LO_XMM, $LO_XMM
vpxor $MI_XMM, $MI_XMM, $MI_XMM
vpxor $HI_XMM, $HI_XMM, $HI_XMM
cmp \$$VL, $DATALEN
jb .Lpartial_vec$local_label_suffix
.Lcrypt_loop_1x$local_label_suffix:
# Process a full vector of length VL.
# Encrypt a vector of counter blocks.
vpshufb $BSWAP_MASK, $LE_CTR, $V0
vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR
vpxord $RNDKEY0, $V0, $V0
lea 16($AESKEY), %rax
.Lvaesenc_loop_tail_full_vec$local_label_suffix:
vbroadcasti32x4 (%rax), $RNDKEY
vaesenc $RNDKEY, $V0, $V0
add \$16, %rax
cmp %rax, $RNDKEYLAST_PTR
jne .Lvaesenc_loop_tail_full_vec$local_label_suffix
vaesenclast $RNDKEYLAST, $V0, $V0
# XOR the data with the vector of keystream blocks.
vmovdqu8 ($SRC), $V1
vpxord $V1, $V0, $V0
vmovdqu8 $V0, ($DST)
# Update GHASH with the ciphertext blocks, without reducing.
vmovdqu8 ($POWERS_PTR), $H_POW1
vpshufb $BSWAP_MASK, @{[ $enc ? $V0 : $V1 ]}, $V0
vpxord $GHASH_ACC, $V0, $V0
@{[ _ghash_mul_noreduce $H_POW1, $V0, $LO, $MI, $HI, $GHASHDATA3,
$V1, $V2, $V3 ]}
vpxor $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
add \$$VL, $POWERS_PTR
add \$$VL, $SRC
add \$$VL, $DST
sub \$$VL, $DATALEN
cmp \$$VL, $DATALEN
jae .Lcrypt_loop_1x$local_label_suffix
test $DATALEN, $DATALEN
jz .Lreduce$local_label_suffix
.Lpartial_vec$local_label_suffix:
# Process a partial vector of length 1 <= DATALEN < VL.
# Set the data mask %k1 to DATALEN 1's.
# Set the key powers mask %k2 to round_up(DATALEN, 16) 1's.
mov \$-1, %rax
bzhi $DATALEN, %rax, %rax
@{[ $VL < 64 ? "kmovd %eax, %k1" : "kmovq %rax, %k1" ]}
add \$15, $DATALEN
and \$-16, $DATALEN
mov \$-1, %rax
bzhi $DATALEN, %rax, %rax
@{[ $VL < 64 ? "kmovd %eax, %k2" : "kmovq %rax, %k2" ]}
# Encrypt one last vector of counter blocks. This does not need to be
# masked. The counter does not need to be incremented here.
vpshufb $BSWAP_MASK, $LE_CTR, $V0
vpxord $RNDKEY0, $V0, $V0
lea 16($AESKEY), %rax
.Lvaesenc_loop_tail_partialvec$local_label_suffix:
vbroadcasti32x4 (%rax), $RNDKEY
vaesenc $RNDKEY, $V0, $V0
add \$16, %rax
cmp %rax, $RNDKEYLAST_PTR
jne .Lvaesenc_loop_tail_partialvec$local_label_suffix
vaesenclast $RNDKEYLAST, $V0, $V0
# XOR the data with the appropriate number of keystream bytes.
vmovdqu8 ($SRC), $V1\{%k1}{z}
vpxord $V1, $V0, $V0
vmovdqu8 $V0, ($DST){%k1}
# Update GHASH with the ciphertext block(s), without reducing.
#
# In the case of DATALEN < VL, the ciphertext is zero-padded to VL.
# (If decrypting, it's done by the above masked load. If encrypting,
# it's done by the below masked register-to-register move.) Note that
# if DATALEN <= VL - 16, there will be additional padding beyond the
# padding of the last block specified by GHASH itself; i.e., there may
# be whole block(s) that get processed by the GHASH multiplication and
# reduction instructions but should not actually be included in the
# GHASH. However, any such blocks are all-zeroes, and the values that
# they're multiplied with are also all-zeroes. Therefore they just add
# 0 * 0 = 0 to the final GHASH result, which makes no difference.
vmovdqu8 ($POWERS_PTR), $H_POW1\{%k2}{z}
@{[ $enc ? "vmovdqu8 $V0, $V1\{%k1}{z}" : "" ]}
vpshufb $BSWAP_MASK, $V1, $V0
vpxord $GHASH_ACC, $V0, $V0
@{[ _ghash_mul_noreduce $H_POW1, $V0, $LO, $MI, $HI, $GHASHDATA3,
$V1, $V2, $V3 ]}
.Lreduce$local_label_suffix:
# Finally, do the GHASH reduction.
@{[ _ghash_reduce $LO, $MI, $HI, $GFPOLY, $V0 ]}
@{[ _horizontal_xor $HI, $HI_XMM, $GHASH_ACC_XMM,
"%xmm0", "%xmm1", "%xmm2" ]}
.Ldone$local_label_suffix:
# Store the updated GHASH accumulator back to memory.
vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
vmovdqu $GHASH_ACC_XMM, ($GHASH_ACC_PTR)
vzeroupper # This is needed after using ymm or zmm registers.
___
return $code;
}
# void gcm_gmult_vpclmulqdq_avx10(uint8_t Xi[16], const u128 Htable[16]);
$code .= _begin_func "gcm_gmult_vpclmulqdq_avx10", 1;
{
my ( $GHASH_ACC_PTR, $H_POWERS ) = @argregs[ 0 .. 1 ];
my ( $GHASH_ACC, $BSWAP_MASK, $H_POW1, $GFPOLY, $T0, $T1, $T2 ) =
map( "%xmm$_", ( 0 .. 6 ) );
$code .= <<___;
@{[ _save_xmmregs (6) ]}
.seh_endprologue
vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC
vmovdqu .Lbswap_mask(%rip), $BSWAP_MASK
vmovdqu $OFFSETOFEND_H_POWERS-16($H_POWERS), $H_POW1
vmovdqu .Lgfpoly(%rip), $GFPOLY
vpshufb $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC
@{[ _ghash_mul $H_POW1, $GHASH_ACC, $GHASH_ACC, $GFPOLY, $T0, $T1, $T2 ]}
vpshufb $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC
vmovdqu $GHASH_ACC, ($GHASH_ACC_PTR)
___
}
$code .= _end_func;
_set_veclen 32;
$code .= _begin_func "gcm_init_vpclmulqdq_avx10", 0;
$code .= _aes_gcm_init;
$code .= _end_func;
$code .= _begin_func "gcm_ghash_vpclmulqdq_avx10_256", 1;
$code .= _ghash_update;
$code .= _end_func;
$code .= _begin_func "aes_gcm_enc_update_vaes_avx10_256", 1;
$code .= _aes_gcm_update 1;
$code .= _end_func;
$code .= _begin_func "aes_gcm_dec_update_vaes_avx10_256", 1;
$code .= _aes_gcm_update 0;
$code .= _end_func;
_set_veclen 64;
$code .= _begin_func "gcm_ghash_vpclmulqdq_avx10_512", 1;
$code .= _ghash_update;
$code .= _end_func;
$code .= _begin_func "aes_gcm_enc_update_vaes_avx10_512", 1;
$code .= _aes_gcm_update 1;
$code .= _end_func;
$code .= _begin_func "aes_gcm_dec_update_vaes_avx10_512", 1;
$code .= _aes_gcm_update 0;
$code .= _end_func;
print $code;
close STDOUT or die "error closing STDOUT: $!";
exit 0;