#!/usr/bin/env perl
# Copyright 2024 The BoringSSL Authors
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#------------------------------------------------------------------------------
#
# VAES and VPCLMULQDQ optimized AES-GCM for x86_64 (AVX2 version)
#
# This is similar to aes-gcm-avx10-x86_64.pl, but it uses AVX2 instead of AVX512
# / AVX10.  This means it can only use 16 vector registers instead of 32, the
# maximum vector length is 32 bytes, and some instructions such as vpternlogd
# and masked loads/stores are unavailable.  However, it is able to run on CPUs
# that have VAES without AVX512 / AVX10, namely AMD Zen 3 (including "Milan"
# server processors) and some Intel client CPUs such as Alder Lake.
#
# This implementation also uses Karatsuba multiplication instead of schoolbook
# multiplication for GHASH in its main loop.  This does not help much on Intel,
# but it improves performance by ~5% on AMD Zen 3 which is the main target for
# this implementation.  Other factors weighing slightly in favor of Karatsuba
# multiplication in this implementation are the lower maximum vector length
# (which means there is space left in the Htable array to cache the halves of
# the key powers XOR'd together) and the unavailability of the vpternlogd
# instruction (which helped schoolbook a bit more than Karatsuba).

use strict;

my $flavour = shift;
my $output  = shift;
if ( $flavour =~ /\./ ) { $output = $flavour; undef $flavour; }

my $win64;
my @argregs;
if ( $flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/ ) {
    $win64   = 1;
    @argregs = ( "%rcx", "%rdx", "%r8", "%r9" );
}
else {
    $win64   = 0;
    @argregs = ( "%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9" );
}

$0 =~ m/(.*[\/\\])[^\/\\]+$/;
my $dir = $1;
my $xlate;
( $xlate = "${dir}x86_64-xlate.pl" and -f $xlate )
  or ( $xlate = "${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate )
  or die "can't locate x86_64-xlate.pl";

open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT = *OUT;

my $g_cur_func_name;
my $g_cur_func_uses_seh;
my @g_cur_func_saved_gpregs;
my @g_cur_func_saved_xmmregs;

sub _begin_func {
    my ( $funcname, $uses_seh ) = @_;
    $g_cur_func_name          = $funcname;
    $g_cur_func_uses_seh      = $uses_seh;
    @g_cur_func_saved_gpregs  = ();
    @g_cur_func_saved_xmmregs = ();
    return <<___;
.globl $funcname
.type $funcname,\@abi-omnipotent
.align 32
$funcname:
    .cfi_startproc
    @{[ $uses_seh ? ".seh_startproc" : "" ]}
    _CET_ENDBR
___
}

# Push a list of general purpose registers onto the stack.
sub _save_gpregs {
    my @gpregs = @_;
    my $code   = "";
    die "_save_gpregs requires uses_seh" unless $g_cur_func_uses_seh;
    die "_save_gpregs can only be called once per function"
      if @g_cur_func_saved_gpregs;
    die "Order must be _save_gpregs, then _save_xmmregs"
      if @g_cur_func_saved_xmmregs;
    @g_cur_func_saved_gpregs = @gpregs;
    for my $reg (@gpregs) {
        $code .= "push $reg\n";
        if ($win64) {
            $code .= ".seh_pushreg $reg\n";
        }
        else {
            $code .= ".cfi_push $reg\n";
        }
    }
    return $code;
}

# Push a list of xmm registers onto the stack if the target is Windows.
sub _save_xmmregs {
    my @xmmregs     = @_;
    my $num_xmmregs = scalar @xmmregs;
    my $code        = "";
    die "_save_xmmregs requires uses_seh" unless $g_cur_func_uses_seh;
    die "_save_xmmregs can only be called once per function"
      if @g_cur_func_saved_xmmregs;
    if ( $win64 and $num_xmmregs > 0 ) {
        @g_cur_func_saved_xmmregs = @xmmregs;
        my $is_misaligned = ( scalar @g_cur_func_saved_gpregs ) % 2 == 0;
        my $alloc_size    = 16 * $num_xmmregs + ( $is_misaligned ? 8 : 0 );
        $code .= "sub \$$alloc_size, %rsp\n";
        $code .= ".seh_stackalloc $alloc_size\n";
        for my $i ( 0 .. $num_xmmregs - 1 ) {
            my $reg_num = $xmmregs[$i];
            my $pos     = 16 * $i;
            $code .= "movdqa %xmm$reg_num, $pos(%rsp)\n";
            $code .= ".seh_savexmm %xmm$reg_num, $pos\n";
        }
    }
    return $code;
}

sub _end_func {
    my $code = "";

    # Restore any xmm registers that were saved earlier.
    my $num_xmmregs = scalar @g_cur_func_saved_xmmregs;
    if ( $win64 and $num_xmmregs > 0 ) {
        my $need_alignment = ( scalar @g_cur_func_saved_gpregs ) % 2 == 0;
        my $alloc_size     = 16 * $num_xmmregs + ( $need_alignment ? 8 : 0 );
        for my $i ( 0 .. $num_xmmregs - 1 ) {
            my $reg_num = $g_cur_func_saved_xmmregs[$i];
            my $pos     = 16 * $i;
            $code .= "movdqa $pos(%rsp), %xmm$reg_num\n";
        }
        $code .= "add \$$alloc_size, %rsp\n";
    }

    # Restore any general purpose registers that were saved earlier.
    for my $reg ( reverse @g_cur_func_saved_gpregs ) {
        $code .= "pop $reg\n";
        if ( !$win64 ) {
            $code .= ".cfi_pop $reg\n";
        }
    }

    $code .= <<___;
    ret
    @{[ $g_cur_func_uses_seh ? ".seh_endproc" : "" ]}
    .cfi_endproc
    .size   $g_cur_func_name, . - $g_cur_func_name
___
    return $code;
}

my $code = <<___;
.section .rodata
.align 16

    # A shuffle mask that reflects the bytes of 16-byte blocks
.Lbswap_mask:
    .quad   0x08090a0b0c0d0e0f, 0x0001020304050607

    # This is the GHASH reducing polynomial without its constant term, i.e.
    # x^128 + x^7 + x^2 + x, represented using the backwards mapping
    # between bits and polynomial coefficients.
    #
    # Alternatively, it can be interpreted as the naturally-ordered
    # representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the
    # "reversed" GHASH reducing polynomial without its x^128 term.
.Lgfpoly:
    .quad   1, 0xc200000000000000

    # Same as above, but with the (1 << 64) bit set.
.Lgfpoly_and_internal_carrybit:
    .quad   1, 0xc200000000000001

.align 32
    # The below constants are used for incrementing the counter blocks.
.Lctr_pattern:
    .quad   0, 0
    .quad   1, 0
.Linc_2blocks:
    .quad   2, 0
    .quad   2, 0

.text
___

# We use Htable[0..7] to store H^8 through H^1, and Htable[8..11] to store the
# 64-bit halves of the key powers XOR'd together (for Karatsuba multiplication)
# in the order 8,6,7,5,4,2,3,1.  We do not use Htable[12..15].
my $NUM_H_POWERS            = 8;
my $OFFSETOFEND_H_POWERS    = $NUM_H_POWERS * 16;
my $OFFSETOF_H_POWERS_XORED = $OFFSETOFEND_H_POWERS;

# Offset to 'rounds' in AES_KEY struct
my $OFFSETOF_AES_ROUNDS = 240;

# GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and store
# the reduced products in \dst.  Uses schoolbook multiplication.
sub _ghash_mul {
    my ( $a, $b, $dst, $gfpoly, $t0, $t1, $t2 ) = @_;
    return <<___;
    vpclmulqdq      \$0x00, $a, $b, $t0        # LO = a_L * b_L
    vpclmulqdq      \$0x01, $a, $b, $t1        # MI_0 = a_L * b_H
    vpclmulqdq      \$0x10, $a, $b, $t2        # MI_1 = a_H * b_L
    vpxor           $t2, $t1, $t1              # MI = MI_0 + MI_1
    vpclmulqdq      \$0x01, $t0, $gfpoly, $t2  # LO_L*(x^63 + x^62 + x^57)
    vpshufd         \$0x4e, $t0, $t0           # Swap halves of LO
    vpxor           $t0, $t1, $t1              # Fold LO into MI (part 1)
    vpxor           $t2, $t1, $t1              # Fold LO into MI (part 2)
    vpclmulqdq      \$0x11, $a, $b, $dst       # HI = a_H * b_H
    vpclmulqdq      \$0x01, $t1, $gfpoly, $t0  # MI_L*(x^63 + x^62 + x^57)
    vpshufd         \$0x4e, $t1, $t1           # Swap halves of MI
    vpxor           $t1, $dst, $dst            # Fold MI into HI (part 1)
    vpxor           $t0, $dst, $dst            # Fold MI into HI (part 2)
___
}

# void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]);
#
# Initialize |Htable| with powers of the GHASH subkey |H|.
#
# We use Htable[0..7] to store H^8 through H^1, and Htable[8..11] to store the
# 64-bit halves of the key powers XOR'd together (for Karatsuba multiplication)
# in the order 8,6,7,5,4,2,3,1.  We do not use Htable[12..15].
$code .= _begin_func "gcm_init_vpclmulqdq_avx2", 1;
{
    my ( $HTABLE, $H_PTR ) = @argregs[ 0 .. 1 ];
    my ( $TMP0,   $TMP0_XMM )   = ( "%ymm0", "%xmm0" );
    my ( $TMP1,   $TMP1_XMM )   = ( "%ymm1", "%xmm1" );
    my ( $TMP2,   $TMP2_XMM )   = ( "%ymm2", "%xmm2" );
    my ( $H_CUR,  $H_CUR_XMM )  = ( "%ymm3", "%xmm3" );
    my ( $H_CUR2, $H_CUR2_XMM ) = ( "%ymm4", "%xmm4" );
    my ( $H_INC,  $H_INC_XMM )  = ( "%ymm5", "%xmm5" );
    my ( $GFPOLY, $GFPOLY_XMM ) = ( "%ymm6", "%xmm6" );

    $code .= <<___;
    @{[ _save_xmmregs (6) ]}
    .seh_endprologue

    # Load the byte-reflected hash subkey.  BoringSSL provides it in
    # byte-reflected form except the two halves are in the wrong order.
    vpshufd         \$0x4e, ($H_PTR), $H_CUR_XMM

    # Finish preprocessing the byte-reflected hash subkey by multiplying it by
    # x^-1 ("standard" interpretation of polynomial coefficients) or
    # equivalently x^1 (natural interpretation).  This gets the key into a
    # format that avoids having to bit-reflect the data blocks later.
    vpshufd         \$0xd3, $H_CUR_XMM, $TMP0_XMM
    vpsrad          \$31, $TMP0_XMM, $TMP0_XMM
    vpaddq          $H_CUR_XMM, $H_CUR_XMM, $H_CUR_XMM
    vpand           .Lgfpoly_and_internal_carrybit(%rip), $TMP0_XMM, $TMP0_XMM
    vpxor           $TMP0_XMM, $H_CUR_XMM, $H_CUR_XMM

    vbroadcasti128  .Lgfpoly(%rip), $GFPOLY

    # Square H^1 to get H^2.
    @{[ _ghash_mul  $H_CUR_XMM, $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM,
                    $TMP0_XMM, $TMP1_XMM, $TMP2_XMM ]}

    # Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2].
    vinserti128     \$1, $H_CUR_XMM, $H_INC, $H_CUR
    vinserti128     \$1, $H_INC_XMM, $H_INC, $H_INC

    # Compute H_CUR2 = [H^4, H^3].
    @{[ _ghash_mul  $H_INC, $H_CUR, $H_CUR2, $GFPOLY, $TMP0, $TMP1, $TMP2 ]}

    # Store [H^2, H^1] and [H^4, H^3].
    vmovdqu         $H_CUR, 3*32($HTABLE)
    vmovdqu         $H_CUR2, 2*32($HTABLE)

    # For Karatsuba multiplication: compute and store the two 64-bit halves of
    # each key power XOR'd together.  Order is 4,2,3,1.
    vpunpcklqdq     $H_CUR, $H_CUR2, $TMP0
    vpunpckhqdq     $H_CUR, $H_CUR2, $TMP1
    vpxor           $TMP1, $TMP0, $TMP0
    vmovdqu         $TMP0, $OFFSETOF_H_POWERS_XORED+32($HTABLE)

    # Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7].
    @{[ _ghash_mul  $H_INC, $H_CUR2, $H_CUR, $GFPOLY, $TMP0, $TMP1, $TMP2 ]}
    @{[ _ghash_mul  $H_INC, $H_CUR, $H_CUR2, $GFPOLY, $TMP0, $TMP1, $TMP2 ]}
    vmovdqu         $H_CUR, 1*32($HTABLE)
    vmovdqu         $H_CUR2, 0*32($HTABLE)

    # Again, compute and store the two 64-bit halves of each key power XOR'd
    # together.  Order is 8,6,7,5.
    vpunpcklqdq     $H_CUR, $H_CUR2, $TMP0
    vpunpckhqdq     $H_CUR, $H_CUR2, $TMP1
    vpxor           $TMP1, $TMP0, $TMP0
    vmovdqu         $TMP0, $OFFSETOF_H_POWERS_XORED($HTABLE)

    vzeroupper
___
}
$code .= _end_func;

# Do one step of the GHASH update of four vectors of data blocks.
#   $i: the step to do, 0 through 9
#   $ghashdata_ptr: pointer to the data blocks (ciphertext or AAD)
#   $htable: pointer to the Htable for the key
#   $bswap_mask: mask for reflecting the bytes of blocks
#   $h_pow[2-1]_xored: XOR'd key powers cached from Htable
#   $tmp[0-2]: temporary registers.  $tmp[1-2] must be preserved across steps.
#   $lo, $mi: working state for this macro that must be preserved across steps
#   $ghash_acc: the GHASH accumulator (input/output)
sub _ghash_step_4x {
    my (
        $i,            $ghashdata_ptr, $htable, $bswap_mask,
        $h_pow2_xored, $h_pow1_xored,  $tmp0,   $tmp0_xmm,
        $tmp1,         $tmp2,          $lo,     $mi,
        $ghash_acc,    $ghash_acc_xmm
    ) = @_;
    my ( $hi, $hi_xmm ) = ( $ghash_acc, $ghash_acc_xmm );    # alias
    if ( $i == 0 ) {
        return <<___;
        # First vector
        vmovdqu         0*32($ghashdata_ptr), $tmp1
        vpshufb         $bswap_mask, $tmp1, $tmp1
        vmovdqu         0*32($htable), $tmp2
        vpxor           $ghash_acc, $tmp1, $tmp1
        vpclmulqdq      \$0x00, $tmp2, $tmp1, $lo
        vpclmulqdq      \$0x11, $tmp2, $tmp1, $hi
        vpunpckhqdq     $tmp1, $tmp1, $tmp0
        vpxor           $tmp1, $tmp0, $tmp0
        vpclmulqdq      \$0x00, $h_pow2_xored, $tmp0, $mi
___
    }
    elsif ( $i == 1 ) {
        return <<___;
___
    }
    elsif ( $i == 2 ) {
        return <<___;
        # Second vector
        vmovdqu         1*32($ghashdata_ptr), $tmp1
        vpshufb         $bswap_mask, $tmp1, $tmp1
        vmovdqu         1*32($htable), $tmp2
        vpclmulqdq      \$0x00, $tmp2, $tmp1, $tmp0
        vpxor           $tmp0, $lo, $lo
        vpclmulqdq      \$0x11, $tmp2, $tmp1, $tmp0
        vpxor           $tmp0, $hi, $hi
        vpunpckhqdq     $tmp1, $tmp1, $tmp0
        vpxor           $tmp1, $tmp0, $tmp0
        vpclmulqdq      \$0x10, $h_pow2_xored, $tmp0, $tmp0
        vpxor           $tmp0, $mi, $mi
___
    }
    elsif ( $i == 3 ) {
        return <<___;
        # Third vector
        vmovdqu         2*32($ghashdata_ptr), $tmp1
        vpshufb         $bswap_mask, $tmp1, $tmp1
        vmovdqu         2*32($htable), $tmp2
___
    }
    elsif ( $i == 4 ) {
        return <<___;
        vpclmulqdq      \$0x00, $tmp2, $tmp1, $tmp0
        vpxor           $tmp0, $lo, $lo
        vpclmulqdq      \$0x11, $tmp2, $tmp1, $tmp0
        vpxor           $tmp0, $hi, $hi
___
    }
    elsif ( $i == 5 ) {
        return <<___;
        vpunpckhqdq     $tmp1, $tmp1, $tmp0
        vpxor           $tmp1, $tmp0, $tmp0
        vpclmulqdq      \$0x00, $h_pow1_xored, $tmp0, $tmp0
        vpxor           $tmp0, $mi, $mi

        # Fourth vector
        vmovdqu         3*32($ghashdata_ptr), $tmp1
        vpshufb         $bswap_mask, $tmp1, $tmp1
___
    }
    elsif ( $i == 6 ) {
        return <<___;
        vmovdqu         3*32($htable), $tmp2
        vpclmulqdq      \$0x00, $tmp2, $tmp1, $tmp0
        vpxor           $tmp0, $lo, $lo
        vpclmulqdq      \$0x11, $tmp2, $tmp1, $tmp0
        vpxor           $tmp0, $hi, $hi
        vpunpckhqdq     $tmp1, $tmp1, $tmp0
        vpxor           $tmp1, $tmp0, $tmp0
        vpclmulqdq      \$0x10, $h_pow1_xored, $tmp0, $tmp0
        vpxor           $tmp0, $mi, $mi
___
    }
    elsif ( $i == 7 ) {
        return <<___;
        # Finalize 'mi' following Karatsuba multiplication.
        vpxor           $lo, $mi, $mi
        vpxor           $hi, $mi, $mi

        # Fold lo into mi.
        vbroadcasti128  .Lgfpoly(%rip), $tmp2
        vpclmulqdq      \$0x01, $lo, $tmp2, $tmp0
        vpshufd         \$0x4e, $lo, $lo
        vpxor           $lo, $mi, $mi
        vpxor           $tmp0, $mi, $mi
___
    }
    elsif ( $i == 8 ) {
        return <<___;
        # Fold mi into hi.
        vpclmulqdq      \$0x01, $mi, $tmp2, $tmp0
        vpshufd         \$0x4e, $mi, $mi
        vpxor           $mi, $hi, $hi
        vpxor           $tmp0, $hi, $hi
___
    }
    elsif ( $i == 9 ) {
        return <<___;
        vextracti128    \$1, $hi, $tmp0_xmm
        vpxor           $tmp0_xmm, $hi_xmm, $ghash_acc_xmm
___
    }
}

sub _ghash_4x {
    my $code = "";
    for my $i ( 0 .. 9 ) {
        $code .= _ghash_step_4x $i, @_;
    }
    return $code;
}

# void gcm_gmult_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16]);
$code .= _begin_func "gcm_gmult_vpclmulqdq_avx2", 1;
{
    my ( $GHASH_ACC_PTR, $HTABLE ) = @argregs[ 0 .. 1 ];
    my ( $GHASH_ACC, $BSWAP_MASK, $H_POW1, $GFPOLY, $T0, $T1, $T2 ) =
      map( "%xmm$_", ( 0 .. 6 ) );

    $code .= <<___;
    @{[ _save_xmmregs (6) ]}
    .seh_endprologue

    vmovdqu         ($GHASH_ACC_PTR), $GHASH_ACC
    vmovdqu         .Lbswap_mask(%rip), $BSWAP_MASK
    vmovdqu         $OFFSETOFEND_H_POWERS-16($HTABLE), $H_POW1
    vmovdqu         .Lgfpoly(%rip), $GFPOLY
    vpshufb         $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC

    @{[ _ghash_mul  $H_POW1, $GHASH_ACC, $GHASH_ACC, $GFPOLY, $T0, $T1, $T2 ]}

    vpshufb         $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC
    vmovdqu         $GHASH_ACC, ($GHASH_ACC_PTR)
___
}
$code .= _end_func;

# void gcm_ghash_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16],
#                                const uint8_t *in, size_t len);
#
# Using the key |Htable|, update the GHASH accumulator |Xi| with the data given
# by |in| and |len|.  |len| must be a multiple of 16.
#
# This function handles large amounts of AAD efficiently, while also keeping the
# overhead low for small amounts of AAD which is the common case.  TLS uses less
# than one block of AAD, but (uncommonly) other use cases may use much more.
$code .= _begin_func "gcm_ghash_vpclmulqdq_avx2", 1;
{
    # Function arguments
    my ( $GHASH_ACC_PTR, $HTABLE, $AAD, $AADLEN ) = @argregs[ 0 .. 3 ];

    # Additional local variables
    my ( $TMP0,       $TMP0_XMM )       = ( "%ymm0", "%xmm0" );
    my ( $TMP1,       $TMP1_XMM )       = ( "%ymm1", "%xmm1" );
    my ( $TMP2,       $TMP2_XMM )       = ( "%ymm2", "%xmm2" );
    my ( $LO,         $LO_XMM )         = ( "%ymm3", "%xmm3" );
    my ( $MI,         $MI_XMM )         = ( "%ymm4", "%xmm4" );
    my ( $GHASH_ACC,  $GHASH_ACC_XMM )  = ( "%ymm5", "%xmm5" );
    my ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( "%ymm6", "%xmm6" );
    my ( $GFPOLY,     $GFPOLY_XMM )     = ( "%ymm7", "%xmm7" );
    my $H_POW2_XORED = "%ymm8";
    my $H_POW1_XORED = "%ymm9";

    $code .= <<___;
    @{[ _save_xmmregs (6 .. 9) ]}
    .seh_endprologue

    vbroadcasti128  .Lbswap_mask(%rip), $BSWAP_MASK
    vmovdqu         ($GHASH_ACC_PTR), $GHASH_ACC_XMM
    vpshufb         $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
    vbroadcasti128  .Lgfpoly(%rip), $GFPOLY

    # Optimize for AADLEN < 32 by checking for AADLEN < 32 before AADLEN < 128.
    cmp             \$32, $AADLEN
    jb              .Lghash_lastblock

    cmp             \$127, $AADLEN
    jbe             .Lghash_loop_1x

    # Update GHASH with 128 bytes of AAD at a time.
    vmovdqu         $OFFSETOF_H_POWERS_XORED($HTABLE), $H_POW2_XORED
    vmovdqu         $OFFSETOF_H_POWERS_XORED+32($HTABLE), $H_POW1_XORED
.Lghash_loop_4x:
    @{[ _ghash_4x   $AAD, $HTABLE, $BSWAP_MASK, $H_POW2_XORED, $H_POW1_XORED,
                    $TMP0, $TMP0_XMM, $TMP1, $TMP2, $LO, $MI, $GHASH_ACC,
                    $GHASH_ACC_XMM ]}
    sub             \$-128, $AAD  # 128 is 4 bytes, -128 is 1 byte
    add             \$-128, $AADLEN
    cmp             \$127, $AADLEN
    ja              .Lghash_loop_4x

    # Update GHASH with 32 bytes of AAD at a time.
    cmp             \$32, $AADLEN
    jb              .Lghash_loop_1x_done
.Lghash_loop_1x:
    vmovdqu         ($AAD), $TMP0
    vpshufb         $BSWAP_MASK, $TMP0, $TMP0
    vpxor           $TMP0, $GHASH_ACC, $GHASH_ACC
    vmovdqu         $OFFSETOFEND_H_POWERS-32($HTABLE), $TMP0
    @{[ _ghash_mul  $TMP0, $GHASH_ACC, $GHASH_ACC, $GFPOLY, $TMP1, $TMP2, $LO ]}
    vextracti128    \$1, $GHASH_ACC, $TMP0_XMM
    vpxor           $TMP0_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
    add             \$32, $AAD
    sub             \$32, $AADLEN
    cmp             \$32, $AADLEN
    jae             .Lghash_loop_1x
.Lghash_loop_1x_done:
    # Issue the vzeroupper that is needed after using ymm registers.  Do it here
    # instead of at the end, to minimize overhead for small AADLEN.
    vzeroupper

    # Update GHASH with the remaining 16-byte block if any.
.Lghash_lastblock:
    test            $AADLEN, $AADLEN
    jz              .Lghash_done
    vmovdqu         ($AAD), $TMP0_XMM
    vpshufb         $BSWAP_MASK_XMM, $TMP0_XMM, $TMP0_XMM
    vpxor           $TMP0_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
    vmovdqu         $OFFSETOFEND_H_POWERS-16($HTABLE), $TMP0_XMM
    @{[ _ghash_mul  $TMP0_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GFPOLY_XMM,
                    $TMP1_XMM, $TMP2_XMM, $LO_XMM ]}

.Lghash_done:
    # Store the updated GHASH accumulator back to memory.
    vpshufb         $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
    vmovdqu         $GHASH_ACC_XMM, ($GHASH_ACC_PTR)
___
}
$code .= _end_func;

sub _vaesenc_4x {
    my ( $round_key, $aesdata0, $aesdata1, $aesdata2, $aesdata3 ) = @_;
    return <<___;
    vaesenc         $round_key, $aesdata0, $aesdata0
    vaesenc         $round_key, $aesdata1, $aesdata1
    vaesenc         $round_key, $aesdata2, $aesdata2
    vaesenc         $round_key, $aesdata3, $aesdata3
___
}

sub _ctr_begin_4x {
    my (
        $le_ctr,   $bswap_mask, $rndkey0,  $aesdata0,
        $aesdata1, $aesdata2,   $aesdata3, $tmp
    ) = @_;
    return <<___;
    # Increment le_ctr four times to generate four vectors of little-endian
    # counter blocks, swap each to big-endian, and store them in aesdata[0-3].
    vmovdqu         .Linc_2blocks(%rip), $tmp
    vpshufb         $bswap_mask, $le_ctr, $aesdata0
    vpaddd          $tmp, $le_ctr, $le_ctr
    vpshufb         $bswap_mask, $le_ctr, $aesdata1
    vpaddd          $tmp, $le_ctr, $le_ctr
    vpshufb         $bswap_mask, $le_ctr, $aesdata2
    vpaddd          $tmp, $le_ctr, $le_ctr
    vpshufb         $bswap_mask, $le_ctr, $aesdata3
    vpaddd          $tmp, $le_ctr, $le_ctr

    # AES "round zero": XOR in the zero-th round key.
    vpxor           $rndkey0, $aesdata0, $aesdata0
    vpxor           $rndkey0, $aesdata1, $aesdata1
    vpxor           $rndkey0, $aesdata2, $aesdata2
    vpxor           $rndkey0, $aesdata3, $aesdata3
___
}

# Do the last AES round for four vectors of counter blocks, XOR four vectors of
# source data with the resulting keystream blocks, and write the result to the
# destination buffer.  The implementation differs slightly as it takes advantage
# of the property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a) to reduce
# latency, but it has the same effect.
sub _aesenclast_and_xor_4x {
    my (
        $src,      $dst,      $rndkeylast, $aesdata0,
        $aesdata1, $aesdata2, $aesdata3,   $t0,
        $t1,       $t2,       $t3
    ) = @_;
    return <<___;
    vpxor           0*32($src), $rndkeylast, $t0
    vpxor           1*32($src), $rndkeylast, $t1
    vpxor           2*32($src), $rndkeylast, $t2
    vpxor           3*32($src), $rndkeylast, $t3
    vaesenclast     $t0, $aesdata0, $aesdata0
    vaesenclast     $t1, $aesdata1, $aesdata1
    vaesenclast     $t2, $aesdata2, $aesdata2
    vaesenclast     $t3, $aesdata3, $aesdata3
    vmovdqu         $aesdata0, 0*32($dst)
    vmovdqu         $aesdata1, 1*32($dst)
    vmovdqu         $aesdata2, 2*32($dst)
    vmovdqu         $aesdata3, 3*32($dst)
___
}

my $g_update_macro_expansion_count = 0;

# void aes_gcm_{enc,dec}_update_vaes_avx2(const uint8_t *in, uint8_t *out,
#                                         size_t len, const AES_KEY *key,
#                                         const uint8_t ivec[16],
#                                         const u128 Htable[16],
#                                         uint8_t Xi[16]);
#
# This macro generates a GCM encryption or decryption update function with the
# above prototype (with \enc selecting which one).  The function computes the
# next portion of the CTR keystream, XOR's it with |len| bytes from |in|, and
# writes the resulting encrypted or decrypted data to |out|.  It also updates
# the GHASH accumulator |Xi| using the next |len| ciphertext bytes.
#
# |len| must be a multiple of 16.  The caller must do any buffering needed to
# ensure this.  Both in-place and out-of-place en/decryption are supported.
#
# |ivec| must give the current counter in big-endian format.  This function
# loads the counter from |ivec| and increments the loaded counter as needed, but
# it does *not* store the updated counter back to |ivec|.  The caller must
# update |ivec| if any more data segments follow.  Internally, only the low
# 32-bit word of the counter is incremented, following the GCM standard.
sub _aes_gcm_update {
    my $local_label_suffix = "__func" . ++$g_update_macro_expansion_count;
    my ($enc)              = @_;
    my $code               = "";

    # Function arguments
    my ( $SRC, $DST, $DATALEN, $AESKEY, $BE_CTR_PTR, $HTABLE, $GHASH_ACC_PTR )
      = $win64
      ? ( @argregs[ 0 .. 3 ], "%rsi", "%rdi", "%r12" )
      : ( @argregs[ 0 .. 5 ], "%r12" );

    # Additional local variables.
    # %rax is used as a temporary register.  BE_CTR_PTR is also available as a
    # temporary register after the counter is loaded.

    # AES key length in bytes
    my ( $AESKEYLEN, $AESKEYLEN64 ) = ( "%r10d", "%r10" );

    # Pointer to the last AES round key for the chosen AES variant
    my $RNDKEYLAST_PTR = "%r11";

    # BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values
    # using vpshufb, copied to all 128-bit lanes.
    my ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( "%ymm0", "%xmm0" );

    # GHASH_ACC is the accumulator variable for GHASH.  When fully reduced,
    # only the lowest 128-bit lane can be nonzero.  When not fully reduced,
    # more than one lane may be used, and they need to be XOR'd together.
    my ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( "%ymm1", "%xmm1" );

    # TMP[0-2] are temporary registers.
    my ( $TMP0, $TMP0_XMM ) = ( "%ymm2", "%xmm2" );
    my ( $TMP1, $TMP1_XMM ) = ( "%ymm3", "%xmm3" );
    my ( $TMP2, $TMP2_XMM ) = ( "%ymm4", "%xmm4" );

    # LO and MI are used to accumulate unreduced GHASH products.
    my ( $LO, $LO_XMM ) = ( "%ymm5", "%xmm5" );
    my ( $MI, $MI_XMM ) = ( "%ymm6", "%xmm6" );

    # Cached key powers from Htable
    my ( $H_POW2_XORED, $H_POW2_XORED_XMM ) = ( "%ymm7", "%xmm7" );
    my ( $H_POW1_XORED, $H_POW1_XORED_XMM ) = ( "%ymm8", "%xmm8" );

    # RNDKEY0 caches the zero-th round key, and RNDKEYLAST the last one.
    my $RNDKEY0    = "%ymm9";
    my $RNDKEYLAST = "%ymm10";

    # LE_CTR contains the next set of little-endian counter blocks.
    my $LE_CTR = "%ymm11";

    # AESDATA[0-3] hold the counter blocks that are being encrypted by AES.
    my ( $AESDATA0, $AESDATA0_XMM ) = ( "%ymm12", "%xmm12" );
    my ( $AESDATA1, $AESDATA1_XMM ) = ( "%ymm13", "%xmm13" );
    my ( $AESDATA2, $AESDATA2_XMM ) = ( "%ymm14", "%xmm14" );
    my ( $AESDATA3, $AESDATA3_XMM ) = ( "%ymm15", "%xmm15" );
    my @AESDATA = ( $AESDATA0, $AESDATA1, $AESDATA2, $AESDATA3 );

    my @ghash_4x_args = (
        $enc ? $DST : $SRC, $HTABLE, $BSWAP_MASK, $H_POW2_XORED,
        $H_POW1_XORED,      $TMP0,   $TMP0_XMM,   $TMP1,
        $TMP2,              $LO,     $MI,         $GHASH_ACC,
        $GHASH_ACC_XMM
    );

    if ($win64) {
        $code .= <<___;
        @{[ _save_gpregs $BE_CTR_PTR, $HTABLE, $GHASH_ACC_PTR ]}
        mov             64(%rsp), $BE_CTR_PTR     # arg5
        mov             72(%rsp), $HTABLE         # arg6
        mov             80(%rsp), $GHASH_ACC_PTR  # arg7
        @{[ _save_xmmregs (6 .. 15) ]}
        .seh_endprologue
___
    }
    else {
        $code .= <<___;
        @{[ _save_gpregs $GHASH_ACC_PTR ]}
        mov             16(%rsp), $GHASH_ACC_PTR  # arg7
___
    }

    if ($enc) {
        $code .= <<___;
#ifdef BORINGSSL_DISPATCH_TEST
        .extern BORINGSSL_function_hit
        movb \$1,BORINGSSL_function_hit+8(%rip)
#endif
___
    }
    $code .= <<___;
    vbroadcasti128  .Lbswap_mask(%rip), $BSWAP_MASK

    # Load the GHASH accumulator and the starting counter.
    # BoringSSL passes these values in big endian format.
    vmovdqu         ($GHASH_ACC_PTR), $GHASH_ACC_XMM
    vpshufb         $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
    vbroadcasti128  ($BE_CTR_PTR), $LE_CTR
    vpshufb         $BSWAP_MASK, $LE_CTR, $LE_CTR

    # Load the AES key length in bytes.  BoringSSL stores number of rounds
    # minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20.
    movl            $OFFSETOF_AES_ROUNDS($AESKEY), $AESKEYLEN
    lea             -20(,$AESKEYLEN,4), $AESKEYLEN

    # Make RNDKEYLAST_PTR point to the last AES round key.  This is the
    # round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
    # respectively.  Then load the zero-th and last round keys.
    lea             6*16($AESKEY,$AESKEYLEN64,4), $RNDKEYLAST_PTR
    vbroadcasti128  ($AESKEY), $RNDKEY0
    vbroadcasti128  ($RNDKEYLAST_PTR), $RNDKEYLAST

    # Finish initializing LE_CTR by adding 1 to the second block.
    vpaddd          .Lctr_pattern(%rip), $LE_CTR, $LE_CTR

    # If there are at least 128 bytes of data, then continue into the loop that
    # processes 128 bytes of data at a time.  Otherwise skip it.
    cmp             \$127, $DATALEN
    jbe             .Lcrypt_loop_4x_done$local_label_suffix

    vmovdqu         $OFFSETOF_H_POWERS_XORED($HTABLE), $H_POW2_XORED
    vmovdqu         $OFFSETOF_H_POWERS_XORED+32($HTABLE), $H_POW1_XORED
___

    # Main loop: en/decrypt and hash 4 vectors (128 bytes) at a time.

    if ($enc) {
        $code .= <<___;
        # Encrypt the first 4 vectors of plaintext blocks.
        @{[ _ctr_begin_4x $LE_CTR, $BSWAP_MASK, $RNDKEY0, @AESDATA, $TMP0 ]}
        lea             16($AESKEY), %rax
.Lvaesenc_loop_first_4_vecs$local_label_suffix:
        vbroadcasti128  (%rax), $TMP0
        @{[ _vaesenc_4x $TMP0, @AESDATA ]}
        add             \$16, %rax
        cmp             %rax, $RNDKEYLAST_PTR
        jne             .Lvaesenc_loop_first_4_vecs$local_label_suffix
        @{[ _aesenclast_and_xor_4x $SRC, $DST, $RNDKEYLAST, @AESDATA,
                                   $TMP0, $TMP1, $LO, $MI ]}
        sub             \$-128, $SRC  # 128 is 4 bytes, -128 is 1 byte
        add             \$-128, $DATALEN
        cmp             \$127, $DATALEN
        jbe             .Lghash_last_ciphertext_4x$local_label_suffix
___
    }

    $code .= <<___;
.align 16
.Lcrypt_loop_4x$local_label_suffix:

    # Start the AES encryption of the counter blocks.
    @{[ _ctr_begin_4x $LE_CTR, $BSWAP_MASK, $RNDKEY0, @AESDATA, $TMP0 ]}
    cmp             \$24, $AESKEYLEN
    jl              .Laes128$local_label_suffix
    je              .Laes192$local_label_suffix
    # AES-256
    vbroadcasti128 -13*16($RNDKEYLAST_PTR), $TMP0
    @{[ _vaesenc_4x $TMP0, @AESDATA ]}
    vbroadcasti128 -12*16($RNDKEYLAST_PTR), $TMP0
    @{[ _vaesenc_4x $TMP0, @AESDATA ]}
.Laes192$local_label_suffix:
    vbroadcasti128 -11*16($RNDKEYLAST_PTR), $TMP0
    @{[ _vaesenc_4x $TMP0, @AESDATA ]}
    vbroadcasti128 -10*16($RNDKEYLAST_PTR), $TMP0
    @{[ _vaesenc_4x $TMP0, @AESDATA ]}
.Laes128$local_label_suffix:
___

    # Prefetch the source data 512 bytes ahead into the L1 data cache, to
    # improve performance when the hardware prefetcher is disabled.  Assumes the
    # L1 data cache line size is 64 bytes (de facto standard on x86_64).
    $code .= "prefetcht0 512($SRC)\n";
    $code .= "prefetcht0 512+64($SRC)\n";

    # Finish the AES encryption of the counter blocks in AESDATA[0-3],
    # interleaved with the GHASH update of the ciphertext blocks.
    for my $i ( reverse 1 .. 9 ) {
        $code .= <<___;
        @{[ _ghash_step_4x 9-$i, @ghash_4x_args ]}
        vbroadcasti128  -$i*16($RNDKEYLAST_PTR), $TMP0
        @{[ _vaesenc_4x $TMP0, @AESDATA ]}
___
    }
    $code .= <<___;
    @{[ _ghash_step_4x 9, @ghash_4x_args ]}

    @{[ $enc ? "sub \$-128, $DST" : "" ]}  # 128 is 4 bytes, -128 is 1 byte
    @{[ _aesenclast_and_xor_4x $SRC, $DST, $RNDKEYLAST, @AESDATA,
                               $TMP0, $TMP1, $LO, $MI ]}
    sub             \$-128, $SRC
    @{[ !$enc ? "sub \$-128, $DST" : "" ]}
    add             \$-128, $DATALEN
    cmp             \$127, $DATALEN
    ja              .Lcrypt_loop_4x$local_label_suffix
___

    if ($enc) {

        # Update GHASH with the last set of ciphertext blocks.
        $code .= <<___;
.Lghash_last_ciphertext_4x$local_label_suffix:
        @{[ _ghash_4x @ghash_4x_args ]}
        sub             \$-128, $DST
___
    }

    my $POWERS_PTR = $BE_CTR_PTR;    # BE_CTR_PTR is free to be reused.
    my ( $HI, $HI_XMM ) = ( $H_POW2_XORED, $H_POW2_XORED_XMM );    # reuse

    $code .= <<___;
.Lcrypt_loop_4x_done$local_label_suffix:
    # Check whether any data remains.
    test            $DATALEN, $DATALEN
    jz              .Ldone$local_label_suffix

    # DATALEN is in [16, 32, 48, 64, 80, 96, 112].

    # Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N
    # is the number of blocks that remain.
    lea             $OFFSETOFEND_H_POWERS($HTABLE), $POWERS_PTR
    sub             $DATALEN, $POWERS_PTR

    # Start collecting the unreduced GHASH intermediate value LO, MI, HI.
    vpxor           $LO_XMM, $LO_XMM, $LO_XMM
    vpxor           $MI_XMM, $MI_XMM, $MI_XMM
    vpxor           $HI_XMM, $HI_XMM, $HI_XMM

    cmp             \$64, $DATALEN
    jb              .Llessthan64bytes$local_label_suffix

    # DATALEN is in [64, 80, 96, 112].  Encrypt two vectors of counter blocks.
    vpshufb         $BSWAP_MASK, $LE_CTR, $AESDATA0
    vpaddd          .Linc_2blocks(%rip), $LE_CTR, $LE_CTR
    vpshufb         $BSWAP_MASK, $LE_CTR, $AESDATA1
    vpaddd          .Linc_2blocks(%rip), $LE_CTR, $LE_CTR
    vpxor           $RNDKEY0, $AESDATA0, $AESDATA0
    vpxor           $RNDKEY0, $AESDATA1, $AESDATA1
    lea             16($AESKEY), %rax
.Lvaesenc_loop_tail_1$local_label_suffix:
    vbroadcasti128  (%rax), $TMP0
    vaesenc         $TMP0, $AESDATA0, $AESDATA0
    vaesenc         $TMP0, $AESDATA1, $AESDATA1
    add             \$16, %rax
    cmp             %rax, $RNDKEYLAST_PTR
    jne             .Lvaesenc_loop_tail_1$local_label_suffix
    vaesenclast     $RNDKEYLAST, $AESDATA0, $AESDATA0
    vaesenclast     $RNDKEYLAST, $AESDATA1, $AESDATA1

    # XOR the data with the two vectors of keystream blocks.
    vmovdqu         0($SRC), $TMP0
    vmovdqu         32($SRC), $TMP1
    vpxor           $TMP0, $AESDATA0, $AESDATA0
    vpxor           $TMP1, $AESDATA1, $AESDATA1
    vmovdqu         $AESDATA0, 0($DST)
    vmovdqu         $AESDATA1, 32($DST)

    # Update GHASH with two vectors of ciphertext blocks, without reducing.
    vpshufb         $BSWAP_MASK, @{[ $enc ? $AESDATA0 : $TMP0 ]}, $AESDATA0
    vpshufb         $BSWAP_MASK, @{[ $enc ? $AESDATA1 : $TMP1 ]}, $AESDATA1
    vpxor           $GHASH_ACC, $AESDATA0, $AESDATA0
    vmovdqu         ($POWERS_PTR), $TMP0
    vmovdqu         32($POWERS_PTR), $TMP1
    vpclmulqdq      \$0x00, $TMP0, $AESDATA0, $LO
    vpclmulqdq      \$0x01, $TMP0, $AESDATA0, $MI
    vpclmulqdq      \$0x10, $TMP0, $AESDATA0, $TMP2
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x11, $TMP0, $AESDATA0, $HI
    vpclmulqdq      \$0x00, $TMP1, $AESDATA1, $TMP2
    vpxor           $TMP2, $LO, $LO
    vpclmulqdq      \$0x01, $TMP1, $AESDATA1, $TMP2
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x10, $TMP1, $AESDATA1, $TMP2
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x11, $TMP1, $AESDATA1, $TMP2
    vpxor           $TMP2, $HI, $HI

    add             \$64, $POWERS_PTR
    add             \$64, $SRC
    add             \$64, $DST
    sub             \$64, $DATALEN
    jz              .Lreduce$local_label_suffix

    vpxor           $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM

    # DATALEN is in [16, 32, 48].  Encrypt two last vectors of counter blocks.
.Llessthan64bytes$local_label_suffix:
    vpshufb         $BSWAP_MASK, $LE_CTR, $AESDATA0
    vpaddd          .Linc_2blocks(%rip), $LE_CTR, $LE_CTR
    vpshufb         $BSWAP_MASK, $LE_CTR, $AESDATA1
    vpxor           $RNDKEY0, $AESDATA0, $AESDATA0
    vpxor           $RNDKEY0, $AESDATA1, $AESDATA1
    lea             16($AESKEY), %rax
.Lvaesenc_loop_tail_2$local_label_suffix:
    vbroadcasti128  (%rax), $TMP0
    vaesenc         $TMP0, $AESDATA0, $AESDATA0
    vaesenc         $TMP0, $AESDATA1, $AESDATA1
    add             \$16, %rax
    cmp             %rax, $RNDKEYLAST_PTR
    jne             .Lvaesenc_loop_tail_2$local_label_suffix
    vaesenclast     $RNDKEYLAST, $AESDATA0, $AESDATA0
    vaesenclast     $RNDKEYLAST, $AESDATA1, $AESDATA1

    # XOR the remaining data with the keystream blocks, and update GHASH with
    # the remaining ciphertext blocks without reducing.

    cmp             \$32, $DATALEN
    jb              .Lxor_one_block$local_label_suffix
    je              .Lxor_two_blocks$local_label_suffix

.Lxor_three_blocks$local_label_suffix:
    vmovdqu         0($SRC), $TMP0
    vmovdqu         32($SRC), $TMP1_XMM
    vpxor           $TMP0, $AESDATA0, $AESDATA0
    vpxor           $TMP1_XMM, $AESDATA1_XMM, $AESDATA1_XMM
    vmovdqu         $AESDATA0, 0($DST)
    vmovdqu         $AESDATA1_XMM, 32($DST)

    vpshufb         $BSWAP_MASK, @{[ $enc ? $AESDATA0 : $TMP0 ]}, $AESDATA0
    vpshufb         $BSWAP_MASK_XMM, @{[ $enc ? $AESDATA1_XMM : $TMP1_XMM ]}, $AESDATA1_XMM
    vpxor           $GHASH_ACC, $AESDATA0, $AESDATA0
    vmovdqu         ($POWERS_PTR), $TMP0
    vmovdqu         32($POWERS_PTR), $TMP1_XMM
    vpclmulqdq      \$0x00, $TMP1_XMM, $AESDATA1_XMM, $TMP2_XMM
    vpxor           $TMP2, $LO, $LO
    vpclmulqdq      \$0x01, $TMP1_XMM, $AESDATA1_XMM, $TMP2_XMM
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x10, $TMP1_XMM, $AESDATA1_XMM, $TMP2_XMM
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x11, $TMP1_XMM, $AESDATA1_XMM, $TMP2_XMM
    vpxor           $TMP2, $HI, $HI
    jmp             .Lghash_mul_one_vec_unreduced$local_label_suffix

.Lxor_two_blocks$local_label_suffix:
    vmovdqu         ($SRC), $TMP0
    vpxor           $TMP0, $AESDATA0, $AESDATA0
    vmovdqu         $AESDATA0, ($DST)
    vpshufb         $BSWAP_MASK, @{[ $enc ? $AESDATA0 : $TMP0 ]}, $AESDATA0
    vpxor           $GHASH_ACC, $AESDATA0, $AESDATA0
    vmovdqu         ($POWERS_PTR), $TMP0
    jmp             .Lghash_mul_one_vec_unreduced$local_label_suffix

.Lxor_one_block$local_label_suffix:
    vmovdqu         ($SRC), $TMP0_XMM
    vpxor           $TMP0_XMM, $AESDATA0_XMM, $AESDATA0_XMM
    vmovdqu         $AESDATA0_XMM, ($DST)
    vpshufb         $BSWAP_MASK_XMM, @{[ $enc ? $AESDATA0_XMM : $TMP0_XMM ]}, $AESDATA0_XMM
    vpxor           $GHASH_ACC_XMM, $AESDATA0_XMM, $AESDATA0_XMM
    vmovdqu         ($POWERS_PTR), $TMP0_XMM

.Lghash_mul_one_vec_unreduced$local_label_suffix:
    vpclmulqdq      \$0x00, $TMP0, $AESDATA0, $TMP2
    vpxor           $TMP2, $LO, $LO
    vpclmulqdq      \$0x01, $TMP0, $AESDATA0, $TMP2
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x10, $TMP0, $AESDATA0, $TMP2
    vpxor           $TMP2, $MI, $MI
    vpclmulqdq      \$0x11, $TMP0, $AESDATA0, $TMP2
    vpxor           $TMP2, $HI, $HI

.Lreduce$local_label_suffix:
    # Finally, do the GHASH reduction.
    vbroadcasti128  .Lgfpoly(%rip), $TMP0
    vpclmulqdq      \$0x01, $LO, $TMP0, $TMP1
    vpshufd         \$0x4e, $LO, $LO
    vpxor           $LO, $MI, $MI
    vpxor           $TMP1, $MI, $MI
    vpclmulqdq      \$0x01, $MI, $TMP0, $TMP1
    vpshufd         \$0x4e, $MI, $MI
    vpxor           $MI, $HI, $HI
    vpxor           $TMP1, $HI, $HI
    vextracti128    \$1, $HI, $GHASH_ACC_XMM
    vpxor           $HI_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM

.Ldone$local_label_suffix:
    # Store the updated GHASH accumulator back to memory.
    vpshufb         $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM
    vmovdqu         $GHASH_ACC_XMM, ($GHASH_ACC_PTR)

    vzeroupper
___
    return $code;
}

$code .= _begin_func "aes_gcm_enc_update_vaes_avx2", 1;
$code .= _aes_gcm_update 1;
$code .= _end_func;

$code .= _begin_func "aes_gcm_dec_update_vaes_avx2", 1;
$code .= _aes_gcm_update 0;
$code .= _end_func;

print $code;
close STDOUT or die "error closing STDOUT: $!";
exit 0;