#! /usr/bin/env perl

# Copyright (c) 2022, ARM Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#========================================================================
# Written by Fangming Fang <fangming.fang@arm.com> for the OpenSSL project,
# derived from https://github.com/ARM-software/AArch64cryptolib, original
# author Samuel Lee <Samuel.Lee@arm.com>.
#========================================================================
#
# Approach - assume we don't want to reload constants, so reserve ~half of
# vector register file for constants
#
# main loop to act on 4 16B blocks per iteration, and then do modulo of the
# accumulated intermediate hashes from the 4 blocks
#
#  ____________________________________________________
# |                                                    |
# | PRE                                                |
# |____________________________________________________|
# |                |                |                  |
# | CTR block 4k+8 | AES block 4k+4 | GHASH block 4k+0 |
# |________________|________________|__________________|
# |                |                |                  |
# | CTR block 4k+9 | AES block 4k+5 | GHASH block 4k+1 |
# |________________|________________|__________________|
# |                |                |                  |
# | CTR block 4k+10| AES block 4k+6 | GHASH block 4k+2 |
# |________________|________________|__________________|
# |                |                |                  |
# | CTR block 4k+11| AES block 4k+7 | GHASH block 4k+3 |
# |________________|____(mostly)____|__________________|
# |                                                    |
# | MODULO                                             |
# |____________________________________________________|
#
# PRE: Ensure previous generated intermediate hash is aligned and merged with
# result for GHASH 4k+0
#
# EXT low_acc, low_acc, low_acc, #8
# EOR res_curr (4k+0), res_curr (4k+0), low_acc
#
# CTR block: Increment and byte reverse counter in scalar registers and transfer
# to SIMD registers
#
# REV     ctr32, rev_ctr32
# ORR     ctr64, constctr96_top32, ctr32, LSL #32
# // Keeping this in scalar registers to free up space in SIMD RF
# INS     ctr_next.d[0], constctr96_bottom64
# INS     ctr_next.d[1], ctr64X
# ADD     rev_ctr32, #1
#
# AES block:
#
# Do AES encryption/decryption on CTR block X and EOR it with input block X.
# Take 256 bytes key below for example. Doing small trick here of loading input
# in scalar registers, EORing with last key and then transferring Given we are
# very constrained in our ASIMD registers this is quite important
#
#     Encrypt:
# LDR     input_low, [ input_ptr  ], #8
# LDR     input_high, [ input_ptr  ], #8
# EOR     input_low, k14_low
# EOR     input_high, k14_high
# INS     res_curr.d[0], input_low
# INS     res_curr.d[1], input_high
# AESE    ctr_curr, k0; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k1; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k2; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k3; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k4; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k5; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k6; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k7; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k8; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k9; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k10; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k11; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k12; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k13
# EOR     res_curr, res_curr, ctr_curr
# ST1     { res_curr.16b  }, [ output_ptr  ], #16
#
#     Decrypt:
# AESE    ctr_curr, k0; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k1; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k2; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k3; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k4; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k5; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k6; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k7; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k8; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k9; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k10; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k11; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k12; AESMC ctr_curr, ctr_curr
# AESE    ctr_curr, k13
# LDR     res_curr, [ input_ptr  ], #16
# EOR     res_curr, res_curr, ctr_curr
# MOV     output_low, res_curr.d[0]
# MOV     output_high, res_curr.d[1]
# EOR     output_low, k14_low
# EOR     output_high, k14_high
# STP     output_low, output_high, [ output_ptr  ], #16
#
# GHASH block X:
#     Do 128b karatsuba polynomial multiplication on block. We only have
#     64b->128b polynomial multipliers, naively that means we need to do 4 64b
#     multiplies to generate a 128b.
#
# multiplication:
#     Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^
#                   (Pmull(Ah,Bl) ^ Pmull(Al,Bh))<<64
#
#     The idea behind Karatsuba multiplication is that we can do just 3 64b
#     multiplies:
#     Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^
#                   (Pmull(Ah^Al,Bh^Bl) ^ Pmull(Ah,Bh) ^
#                   Pmull(Al,Bl))<<64
#
#     There is some complication here because the bit order of GHASH's PMULL is
#     reversed compared to elsewhere, so we are multiplying with "twisted"
#     powers of H
#
# Note: We can PMULL directly into the acc_x in first GHASH of the loop
#
# Note: For scheduling big cores we want to split the processing to happen over
#       two loop iterations - otherwise the critical path latency dominates the
#       performance.
#
#       This has a knock on effect on register pressure, so we have to be a bit
#       more clever with our temporary registers than indicated here
#
# REV64   res_curr, res_curr
# INS     t_m.d[0], res_curr.d[1]
# EOR     t_m.8B, t_m.8B, res_curr.8B
# PMULL2  t_h, res_curr, HX
# PMULL   t_l, res_curr, HX
# PMULL   t_m, t_m, HX_k
# EOR     acc_h, acc_h, t_h
# EOR     acc_l, acc_l, t_l
# EOR     acc_m, acc_m, t_m
#
# MODULO: take the partial accumulators (~representing sum of 256b
#         multiplication results), from GHASH and do modulo reduction on them
#         There is some complication here because the bit order of GHASH's
#         PMULL is reversed compared to elsewhere, so we are doing modulo with
#         a reversed constant
#
# EOR     acc_m, acc_m, acc_h
# EOR     acc_m, acc_m, acc_l                // Finish off karatsuba processing
# PMULL   t_mod, acc_h, mod_constant
# EXT     acc_h, acc_h, acc_h, #8
# EOR     acc_m, acc_m, acc_h
# EOR     acc_m, acc_m, t_mod
# PMULL   acc_h, acc_m, mod_constant
# EXT     acc_m, acc_m, acc_m, #8
# EOR     acc_l, acc_l, acc_h
# EOR     acc_l, acc_l, acc_m
#
# Key Optimizations:
#
# NOTE: This implementation is heavily NEON-bound due to the large amount of
# vector computation. The original code from which this was derived often
# avoided unnecessary Load/Store operations, but at the expense of
# increased NEON pipeline pressure and more GPR <-> NEON data movements.
# The primary goal of the optimizations applied here is to reduce NEON
# pipeline pressure. This includes minimizing NEON register-to-register moves,
# GPR to NEON transfers (and vice versa), and other operations that consume
# execution slots on the NEON pipes.
#
# 1.  Merged Kernels: AES-128, AES-192, and AES-256 GCM encryption and
#     decryption are handled by single kernels each, reducing code size.
#     Conditional branching is used for the key-size specific rounds.
#
# 2.  Aggressive Pipelining: GHASH and AES operations for different blocks
#     (4k to 4k+3) are heavily interleaved within the main loop. GHASH for
#     block `n` starts while AES for block `n` is still in progress, and
#     AES for block `n+4` begins. This hides instruction latencies.
#
# 3.  Optimized Counter Handling: Instead of incrementing, reversing, and
#     moving the counter to NEON registers for each of the four blocks in
#     every loop iteration, we precompute and cache counter values on the
#     stack.
#     - The lower 96 bits of the counter are constant across the four blocks
#       within an iteration.
#     - The upper 32 bits (the byte-swapped incrementing part) are calculated
#       for all four blocks (N, N+1, N+2, N+3) and stored on the stack at
#       the start of the loop.
#     - Values are calculated one loop iteration ahead of time. That is,
#       immediately after loading values N-N+3 we calculate and store values
#       N+4-N+7. This ensures that the data is available when loaded without
#       store-load forwarding.
#     - This strategy eliminates repeated scalar-to-vector transfers (FMOV)
#       and scalar increment/reverse operations inside the tight loop.
#
# 4.  Reduced Scalar-NEON Transfers:
#     - For encryption, the final XOR with the last round key (rkN) and the
#       AES-encrypted counter is done entirely in NEON registers using
#       EOR or PLATFORM_EOR3, avoiding costly FMOV instructions to move
#       plaintext data from GPRs to NEON after partial operations. A similar
#       optimization is used for decryption.
#
# 5.  EOR3 Optimization: The PLATFORM_EOR3 macro allows using the single
#     EOR3 instruction (if available) to replace two EOR instructions,
#     i.e., D = A ^ B ^ C. This is used in both GHASH accumulator updates
#     and the final AES output XOR. Both versions of encrypt and decrypt are
#     stored in the same assembly file.
#
# 6.  Instruction scheduling was significantly modified both to directly improve
#     performance as well as to remove false dependencies when that allowed
#     additional improvements. There is still a bit of additional speedup
#     possible but would be easiest to extract with platform-specific
#     implementations.
#
use strict;
use warnings;

my $flavour = shift;
my $output  = shift;

if (!defined $flavour) {
    die "Usage: $0 <flavour> [output_filename]\n";
}

$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1;
my $xlate;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT, "|-", $^X, $xlate, $flavour, $output;
*STDOUT=*OUT;

# Converts an EOR3 pseudo-instruction into either the raw EOR3 encoding or into
# two EOR2 instructions.
sub process_eor3_match {
    my ($vd_str, $vn_str, $vm_str, $va_str, $is_eor3) = @_;

    if ($is_eor3) {
        # Directly emit the EOR3 mnemonic.
        # Requires: .arch armv8.2-a+sha3 OR -march=armv8.2-a+sha3
        return sprintf("    eor3    %s.16b, %s.16b, %s.16b, %s.16b",
                       $vd_str, $vn_str, $vm_str, $va_str);
    } else {
        # Fallback: Two standard NEON EOR instructions
        # Result = (Vn ^ Vm) ^ Va
        # 1. Vd = Vn ^ Vm
        # 2. Vd = Vd ^ Va
        return sprintf("    eor     %s.16b, %s.16b, %s.16b\n" .
                       "    eor     %s.16b, %s.16b, %s.16b",
                       $vd_str, $vn_str, $vm_str,   # Step 1
                       $vd_str, $vd_str, $va_str);  # Step 2
    }
}

# Generates the code string for a specific version (Standard or EOR3).
# Performs regex substitutions for EOR3 instructions and symbol renaming.
sub get_transformed_code {
    my ($is_eor3, $template) = @_;
    my $temp_code = $template;

    if ($is_eor3) {
        # Suffix function names
        $temp_code =~ s/\b(aes_gcm_enc_kernel)\b/${1}_eor3/g;
        $temp_code =~ s/\b(aes_gcm_dec_kernel)\b/${1}_eor3/g;
        # Suffix local labels
        $temp_code =~ s/(\.L[a-zA-Z0-9_]+)/$1_eor3/g;
    }

    $temp_code =~ s/PLATFORM_EOR3\s*\(\s*([vV0-9]+)\s*,\s*([vV0-9]+)\s*,\s*([vV0-9]+)\s*,\s*([vV0-9]+)\s*\)/
        process_eor3_match($1, $2, $3, $4, $is_eor3)/ge;

    return $temp_code;
}

{
{
    # We attempt to minimize the number of aliases for the same register but
    # unfortunately sometimes you have to specify a Q or D register.

    ## AArch64 General-Purpose Registers (GPRs)
    my $input_ptr          = "x0";     # IN: Pointer to input plaintext
    my $bit_length         = "x1";     # IN: Length of input in bits
    my $output_ptr         = "x2";     # IN: Pointer to output ciphertext
    my $current_tag        = "x3";     # IN: Pointer to 16-byte GCM tag (T)
    my $end_input_ptr      = "x4";     # Calculated end of all input
    my $main_end_input_ptr = "x5";     # Calculated end of the main 4-block loop
    my $Htable             = "x6";     # Pointer to GHASH key table (H)
    my $key_ptr            = "x7";     # Pointer to end of AES key schedule
    my $cc                 = "x8";     # Pointer to AES key schedule (context)

    # Re-purposed registers for tail/scalar handling
    my $input_l0           = "x6";
    my $input_h0           = "x7";

    my $T32                = "x10";    # Holds scalar 64-bit counter value
    my $W32                = "w10";    # Holds scalar 32-bit counter value
    my $W_T32              = "w12";    # Holds reversed scalar 32-bit counter
    my $rkN_l              = "x13";    # Holds low 64 bits of final round key
    my $rkN_h              = "x14";    # Holds high 64 bits of final round key
    my $len                = "x15";    # Byte length of input
    my $counter            = "x16";    # Holds pointer to the 16-byte counter block (CTR)
    my $rounds             = "x17";    # Number of AES rounds (scalar)
    my $rounds_w           = "w17";    # Number of AES rounds (scalar)
    my $tmp_gpr_w          = "w20";    # Scratch register (word)
    my $mod_constantx      = "x21";    # Holds GHASH modulus 0xc2... (scalar)

    ## NEON/SIMD Registers (v0-v31)
    my ($ctr0, $ctr1, $ctr2, $ctr3)     = map("v$_", (0..3));
    my ($ctr0q, $ctr1q, $ctr2q, $ctr3q) = map("q$_", (0..3));
    my $ctr0d                           = "d0";

    my ($res0, $res1, $res2, $res3)     = map("v$_", (4..7));
    my ($res0q, $res1q, $res2q, $res3q) = map("q$_", (4..7));
    my ($res0d, $res1d, $res2d, $res3d) = map("d$_", (4..7));

    my ($acc_h, $acc_m, $acc_l)         = map("v$_", (9..11));

    my ($h1, $h2, $h3, $h4)             = map("v$_", (12..15));
    my ($h1q, $h2q, $h3q, $h4q)         = map("q$_", (12..15));

    my ($h12k, $h34k)                   = map("v$_", (16..17)); # H[1]^H[2] and H[3]^H[4]

    my ($rk0, $rk1, $rk2, $rk3, $rk4, $rk5, $rk6, $rk7, $rk8) = map("v$_", (18..26));
    my ($rk0q, $rk1q, $rk2q, $rk3q, $rk4q, $rk5q, $rk6q, $rk7q, $rk8q) = map("q$_", (18..26));

    my $rk9_11_tmp         = "v27";    # Used for rk9 (if AES-192) and rk11 (if AES-256)
    my $rk10_12            = "v28";    # Used for rk10 (if AES-192) and rk12 (if AES-256)
    my $rk9_11_tmpq        = "q27";
    my $rk10_12q           = "q28";

    my $rkNm1              = "v31";    # Round N-1 key
    my $rkNm1q             = "q31";

    my $mod_constant       = "v8";
    my $mod_constantd      = "d8";

    my $final_block_dest   = "v18";
    my $final_block_destq  = "q18";

    my ($ghash_t0, $ghash_t1, $ghash_t2, $ghash_t3)     = map("v$_", (20..23));
    my ($ghash_t0d, $ghash_t1d, $ghash_t2d, $ghash_t3d) = map("d$_", (20..23));

    my $v_rkN              = "v30";
    my $v_rkNq             = "q30";

    # Locations on the stack
    my $mod_constant_sp_offset = 128;
    my $ctr0_sp_offset         = 160;
    my $ctr1_sp_offset         = 176;
    my $ctr2_sp_offset         = 192;
    my $ctr3_sp_offset         = 208;

    # Registers specific to dec_kernel
    my ($input_l1, $input_h1)  = map("x$_", (19..20));
    my $ctr32w                 = "w9";
    my ($ctr32x, $ctr96_b64x)  = map("x$_", (9..10));
    my $acc_md                 = "d10";

    # Decryption temporary NEON registers
    my $t0  = "v8";   my $t0d  = "d8";
    my $t1  = "v4";   my $t1d  = "d4";
    my $t2  = "v8";   my $t2d  = "d8";
    my $t3  = "v4";   my $t3d  = "d4";
    my $t4  = "v4";   my $t4d  = "d4";
    my $t5  = "v5";   my $t5d  = "d5";
    my $t6  = "v8";   my $t6d  = "d8";
    my $t7  = "v5";   my $t7d  = "d5";
    my $t8  = "v4";   my $t8d  = "d8";
    my $t9  = "v6";   my $t9d  = "d6";

    my ($ctr_t0, $ctr_t1, $ctr_t2, $ctr_t3) = map("v$_", (4..7));
    my $mod_t = "v7";

    # Decryption key registers
    my $rk2q1 = "v20.1q";
    my $rk3q1 = "v21.1q";
    my $rk4v  = "v22";
    my $rk4d  = "d22";

    my ($output_l1, $output_h1, $output_l2, $output_h2, $output_l3, $output_h3) = map("x$_", (19..24));
    my ($output_l0, $output_h0) = map("x$_", (6..7));

    # --- Assemble the entire template ---
    my $code_template = "";
    my $header_directives = <<'___';
#if __ARM_MAX_ARCH__ >= 8
.arch armv8.2-a+crypto+sha3
.text
___

    $code_template .= <<"___";
.global aes_gcm_enc_kernel
.type   aes_gcm_enc_kernel,%function
.align  4
aes_gcm_enc_kernel:
    AARCH64_SIGN_LINK_REGISTER
    stp    x29, x30, [sp, #-224]!
    mov    x29, sp
    ld1    { $ctr0.16b}, [x4]                                                 // Load initial counter block
    stp    x19, x20, [sp, #16]
    mov    $ctr1.16b, $ctr0.16b                                               // Initialize ctr1-3 from ctr0
    mov    $ctr2.16b, $ctr0.16b
    mov    $ctr3.16b, $ctr0.16b
    mov    $counter, x4                                                       // Pointer to counter block in memory
    mov    $cc, x5                                                            // Pointer to AES key schedule context
    stp    $mod_constantx, x22, [sp, #32]
    // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel
    stp    d8, d9, [sp, #64]                                                  // Save Neon registers
    stp    d10, d11, [sp, #80]
    stp    d12, d13, [sp, #96]
    stp    d14, d15, [sp, #112]
    ldr    $rounds_w, [$cc, #240]                                             // Load number of AES rounds
    add    $key_ptr, $cc, $rounds, lsl #4                                     // Calculate pointer to the last round key
    ldp    $rkN_l, $rkN_h, [$key_ptr]                                         // load round N key (for final XOR)
    ldr    $rkNm1q, [$key_ptr, #-16]                                          // load round N-1 key
    add    $end_input_ptr, $input_ptr, $bit_length, lsr #3                    // Calculate end of input
    lsr    $main_end_input_ptr, $bit_length, #3                               // Total byte length
    mov    $len, $main_end_input_ptr
    ldr    $W_T32, [$counter, #12]                                            // Load counter's low 32 bits
    sub    $main_end_input_ptr, $main_end_input_ptr, #1                       // byte_len - 1
    ldr    $rk0q, [$cc, #0]                                                   // load rk0
    and    $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0      // Align main loop end to a multiple of 64 bytes
    add    $main_end_input_ptr, $main_end_input_ptr, $input_ptr
    rev    $W_T32, $W_T32                                                     // Reverse for big-endian increment
    uxtw   $T32, $W_T32                                                       // Zero extend reversed w12 into x10 for final counter update
    // Pre-compute this value instead of using two instructions to reconstruct it every iteration
    mov    $mod_constantx, #0xc200000000000000                                // GHASH reduction constant
    str    $mod_constantx, [sp, #$mod_constant_sp_offset]
    // We maintain four copies of ctr values on the stack. Each loop iteration we
    // store the updated ctr value to the last four bytes (e.g., $ctr0_sp_offset + 12).
    // We then load the four values. This avoids a singificant number of
    // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we
    // calculate and store the values one iteration ahead so they have time to
    // drain before we load them.
    str    $ctr0q,     [sp, #$ctr0_sp_offset]                                  // Store base counter for block 0-3
    str    $ctr0q,     [sp, #$ctr1_sp_offset]
    str    $ctr0q,     [sp, #$ctr2_sp_offset]
    str    $ctr0q,     [sp, #$ctr3_sp_offset]
    // Since we need the values right away don't go through the stack this first
    // time. Manually insert the incremented big-endian counter values.
    rev    $tmp_gpr_w, $W_T32
    mov    $ctr0.s[3], $tmp_gpr_w                                             // ctr0 + 0
    add    $tmp_gpr_w, $W_T32, #1
    rev    $tmp_gpr_w, $tmp_gpr_w
    mov    $ctr1.s[3], $tmp_gpr_w                                             // ctr0 + 1
    add    $tmp_gpr_w, $W_T32, #2
    rev    $tmp_gpr_w, $tmp_gpr_w
    mov    $ctr2.s[3], $tmp_gpr_w                                             // ctr0 + 2
    add    $tmp_gpr_w, $W_T32, #3
    rev    $tmp_gpr_w, $tmp_gpr_w
    mov    $ctr3.s[3], $tmp_gpr_w                                             // ctr0 + 3
    // Calculate the ctr values for the *next* (not current) group of four
    // blocks. Store the incremented parts to the stack.
    add    $tmp_gpr_w, $W_T32, #4
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}]                             // ctr0 + 4 for next iter
    add    $tmp_gpr_w, $W_T32, #5
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}]                             // ctr0 + 5 for next iter
    add    $tmp_gpr_w, $W_T32, #6
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}]                             // ctr0 + 6 for next iter
    add    $tmp_gpr_w, $W_T32, #7
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}]                             // ctr0 + 7 for next iter
    add    $W_T32, $W_T32, #8                                                 // Advance counter past these two sets
    // --- Start AES for first 4 blocks ---
    aese    $ctr0.16b, $rk0.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 0
    ldp    $rk1q, $rk2q, [$cc, #16]                                           // load rk1, rk2
    aese    $ctr1.16b, $rk0.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 0
    ldp    $rk3q, $rk4q, [$cc, #48]                                           // load rk3, rk4
    aese    $ctr2.16b, $rk0.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 0
    ldp    $rk5q, $rk6q, [$cc, #80]                                           // load rk5, rk6
    aese    $ctr3.16b, $rk0.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 0
    ldp    $h2q, $h3q, [$Htable, #32]                                         // load H2, H3 (GHASH keys)
    aese    $ctr0.16b, $rk1.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 1
    ldp    $rk7q, $rk8q, [$cc, #112]                                          // load rk7, rk8
    aese    $ctr1.16b, $rk1.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 1
    aese    $ctr2.16b, $rk1.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 1
    aese    $ctr3.16b, $rk1.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 1
    aese    $ctr0.16b, $rk2.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 2
    ext    $h3.16b, $h3.16b, $h3.16b, #8                                      // Byte swap H3 for GHASH
    aese    $ctr1.16b, $rk2.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 2
    ext    $h2.16b, $h2.16b, $h2.16b, #8                                      // Byte swap H2 for GHASH
    aese    $ctr2.16b, $rk2.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 2
    ldr    $h4q, [$Htable, #80]                                               // load H4
    aese    $ctr3.16b, $rk2.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 2
    ext    $h4.16b, $h4.16b, $h4.16b, #8                                      // Byte swap H4 for GHASH
    aese    $ctr0.16b, $rk3.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 3
    ld1    { $acc_l.16b}, [$current_tag]                                      // Load initial GHASH accumulator (T)
    aese    $ctr1.16b, $rk3.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 3
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8                             // Byte swap T for GHASH
    aese    $ctr2.16b, $rk3.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 3
    rev64    $acc_l.16b, $acc_l.16b                                           // Correct byte order within 64-bit lanes
    aese    $ctr3.16b, $rk3.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 3
    trn2    $h34k.2d,  $h3.2d,    $h4.2d                                      // Karatsuba key: H4_low | H3_low
    aese    $ctr0.16b, $rk4.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 4
    ldr    $h1q, [$Htable]                                                    // load H1
    aese    $ctr1.16b, $rk4.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 4
    ext    $h1.16b, $h1.16b, $h1.16b, #8                                      // Byte swap H1 for GHASH
    aese    $ctr2.16b, $rk4.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 4
    trn1    $acc_h.2d, $h3.2d,    $h4.2d                                      // Karatsuba key: H4_high | H3_high
    aese    $ctr3.16b, $rk4.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 4
    trn2    $h12k.2d,  $h1.2d,    $h2.2d                                      // Karatsuba key: H2_low | H1_low
    aese    $ctr0.16b, $rk5.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 5
    ldr    $v_rkNq, [$key_ptr]                                                // Preload round N key for final EOR
    aese    $ctr1.16b, $rk5.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 5
    aese    $ctr3.16b, $rk5.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 5
    aese    $ctr2.16b, $rk5.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 5
    aese    $ctr0.16b, $rk6.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 6
    aese    $ctr1.16b, $rk6.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 6
    aese    $ctr2.16b, $rk6.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 6
    aese    $ctr3.16b, $rk6.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 6
    aese    $ctr0.16b, $rk7.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 7
    aese    $ctr1.16b, $rk7.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 7
    aese    $ctr2.16b, $rk7.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 7
    aese    $ctr3.16b, $rk7.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 7
    aese    $ctr0.16b, $rk8.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 8
    aese    $ctr1.16b, $rk8.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 8
    aese    $ctr2.16b, $rk8.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 8
    aese    $ctr3.16b, $rk8.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 8
    cmp    $rounds, #12                                                       // setup flags for AES-128/192/256 check
    b.lt    .Lenc_finish_first_blocks                                         // branch if AES-128
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #144]                               // load rk9, rk10
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 9
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 9
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 9
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 9
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 10
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 10
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 10
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 10
    b.eq    .Lenc_finish_first_blocks                                         // branch if AES-192
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #176]                               // load rk11, rk12
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 11
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 11
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 11
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 11
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 12
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 12
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 12
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 12
.Lenc_finish_first_blocks:
    cmp    $input_ptr, $main_end_input_ptr                                    // check if we have <= 4 blocks to process in the tail
    eor    $h34k.16b, $h34k.16b, $acc_h.16b                                   // Karatsuba key: H3^H4
    aese    $ctr0.16b, $rkNm1.16b                                             // AES block 0 - round N-1
    aese    $ctr1.16b, $rkNm1.16b                                             // AES block 1 - round N-1
    aese    $ctr2.16b, $rkNm1.16b                                             // AES block 2 - round N-1
    aese    $ctr3.16b, $rkNm1.16b                                             // AES block 3 - round N-1
    trn1    $mod_constant.2d,  $h1.2d, $h2.2d                                 // Karatsuba key: H2_high | H1_high
    eor    $h12k.16b, $h12k.16b, $mod_constant.16b                            // Karatsuba key: H1^H2
    b.ge    .Lenc_tail                                                        // handle tail if no more full 4-block sets
    ldp    $res2q, $res3q, [$input_ptr, #32]                                  // AES blocks 2,3 load plaintext
    ldp    $res0q, $res1q, [$input_ptr], #64                                  // AES blocks 0,1 load plaintext
    // Compute and store first 4 ciphertext blocks
    PLATFORM_EOR3($res0, $res0, $v_rkN, $ctr0)                                // AES block 0 - result = PT ^ AES(ctr0)
    PLATFORM_EOR3($res1, $res1, $v_rkN, $ctr1)                                // AES block 1 - result = PT ^ AES(ctr1)
    PLATFORM_EOR3($res2, $res2, $v_rkN, $ctr2)                                // AES block 2 - result = PT ^ AES(ctr2)
    PLATFORM_EOR3($res3, $res3, $v_rkN, $ctr3)                                // AES block 3 - result = PT ^ AES(ctr3)
    st1    { $res0.16b, $res1.16b, $res2.16b, $res3.16b}, [$output_ptr], #64  // AES blocks 0-3 - store result
    // Load counter values for the second iteration from the stack
    ldp    $ctr0q, $ctr1q, [sp, #$ctr0_sp_offset]
    ldp    $ctr2q, $ctr3q, [sp, #$ctr2_sp_offset]
    // Prepare and store counter values for the third iteration
    rev    $tmp_gpr_w, $W_T32
    str    $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}]                             // ctr + 8
    add    $tmp_gpr_w, $W_T32, #1
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}]                             // ctr + 9
    add    $tmp_gpr_w, $W_T32, #2
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}]                             // ctr + 10
    add    $tmp_gpr_w, $W_T32, #3
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}]                             // ctr + 11
    add    $W_T32, $W_T32, #4                                                 // Advance counter base
    cmp    $input_ptr, $main_end_input_ptr                                    // check if we have <= 4 blocks remaining
    b.ge    .Lenc_prepretail                                                  // go to prepretail if < 2 full loops left
.Lenc_main_loop:    //    main loop start (processes 4 blocks per iteration)
    // --- AES Pipeline for blocks 4k+4 to 4k+7 ---
    aese    $ctr0.16b, $rk0.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 0
    aese    $ctr1.16b, $rk0.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 0
    aese    $ctr2.16b, $rk0.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 0
    aese    $ctr3.16b, $rk0.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 0
    ldr    $mod_constantd, [sp, #$mod_constant_sp_offset]                      // Load GHASH reduction constant
    aese    $ctr0.16b, $rk1.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 1
    aese    $ctr1.16b, $rk1.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 1
    aese    $ctr2.16b, $rk1.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 1
    aese    $ctr3.16b, $rk1.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 1
    // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 ---
    rev64    $res0.16b, $res0.16b                                             // GHASH block 4k - Byte swap CT
    rev64    $res1.16b, $res1.16b                                             // GHASH block 4k+1 - Byte swap CT
    rev64    $res2.16b, $res2.16b                                             // GHASH block 4k+2 - Byte swap CT
    rev64    $res3.16b, $res3.16b                                             // GHASH block 4k+3 - Byte swap CT
    aese    $ctr0.16b, $rk2.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 2
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8                             // GHASH - prepare acc for XOR
    aese    $ctr1.16b, $rk2.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 2
    eor    $res0.16b, $res0.16b, $acc_l.16b                                   // GHASH block 4k - Y_i = CT_i ^ Y_{i-1}
    aese    $ctr2.16b, $rk2.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 2
    pmull    $acc_l.1q, $res0.1d, $h4.1d                                      // GHASH block 4k - low
    aese    $ctr3.16b, $rk2.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 2
    pmull2    $acc_h.1q, $res2.2d, $h2.2d                                     // GHASH block 4k+2 - high
    aese    $ctr0.16b, $rk3.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 3
    mov    $acc_md, $h34k.d[1]                                                // GHASH block 4k - mid Karatsuba key
    aese    $ctr1.16b, $rk3.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 3
    mov    $ghash_t0d, $res0.d[1]                                             // GHASH block 4k - mid
    aese    $ctr2.16b, $rk3.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 3
    eor    $ghash_t0.8b, $ghash_t0.8b, $res0.8b                               // GHASH block 4k - mid
    aese    $ctr3.16b, $rk3.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 3
    mov    $ghash_t1d, $res1.d[1]                                             // GHASH block 4k+1 - mid
    aese    $ctr0.16b, $rk4.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 4
    eor    $ghash_t1.8b, $ghash_t1.8b, $res1.8b                               // GHASH block 4k+1 - mid
    aese    $ctr1.16b, $rk4.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 4
    pmull    $acc_m.1q, $ghash_t0.1d, $acc_m.1d                               // GHASH block 4k - mid
    aese    $ctr2.16b, $rk4.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 4
    pmull    $ghash_t1.1q, $ghash_t1.1d, $h34k.1d                             // GHASH block 4k+1 - mid
    aese    $ctr3.16b, $rk4.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 4
    eor    $acc_m.16b, $acc_m.16b, $ghash_t1.16b                              // GHASH block 4k+1 - mid
    aese    $ctr0.16b, $rk5.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 5
    ext    $ghash_t2.16b, $ghash_t2.16b, $res2.16b, #8                        // GHASH block 4k+2 - mid
    aese    $ctr1.16b, $rk5.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 5
    eor    $ghash_t2.16b, $ghash_t2.16b, $res2.16b                            // GHASH block 4k+2 - mid
    aese    $ctr2.16b, $rk5.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 5
    pmull2    $ghash_t2.1q, $ghash_t2.2d, $h12k.2d                            // GHASH block 4k+2 - mid
    aese    $ctr3.16b, $rk5.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 5
    mov    $ghash_t3d, $res3.d[1]                                             // GHASH block 4k+3 - mid
    aese    $ctr0.16b, $rk6.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 6
    eor    $ghash_t3.8b, $ghash_t3.8b, $res3.8b                               // GHASH block 4k+3 - mid
    aese    $ctr1.16b, $rk6.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 6
    pmull    $ghash_t3.1q, $ghash_t3.1d, $h12k.1d                             // GHASH block 4k+3 - mid
    aese    $ctr2.16b, $rk6.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 6
    PLATFORM_EOR3($acc_m, $acc_m, $ghash_t2, $ghash_t3)                       // GHASH block 4k+2/3 - mid
    aese    $ctr3.16b, $rk6.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 6
    pmull2    $ghash_t2.1q, $res0.2d, $h4.2d                                  // GHASH block 4k - high
    aese    $ctr0.16b, $rk7.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 7
    pmull2    $ghash_t1.1q, $res3.2d, $h1.2d                                  // GHASH block 4k+3 - high
    eor    $ghash_t2.16b, $ghash_t2.16b, $ghash_t1.16b                        // GHASH block 4k+3 - high
    aese    $ctr1.16b, $rk7.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 7
    pmull2    $ghash_t3.1q, $res1.2d, $h3.2d                                  // GHASH block 4k+1 - high
    aese    $ctr2.16b, $rk7.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 7
    pmull    $ghash_t1.1q, $res2.1d, $h2.1d                                   // GHASH block 4k+2 - low
    aese    $ctr3.16b, $rk7.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 7
    pmull    $ghash_t0.1q, $res1.1d, $h3.1d                                   // GHASH block 4k+1 - low
    PLATFORM_EOR3($acc_h, $acc_h, $ghash_t2, $ghash_t3)                       // GHASH block 4k/1/2/3 - high
    pmull    $ghash_t2.1q, $res3.1d, $h1.1d                                   // GHASH block 4k+3 - low
    ldp    $res2q, $res3q, [$input_ptr, #32]
    ldp    $res0q, $res1q, [$input_ptr], #64
    eor    $ghash_t0.16b, $ghash_t0.16b, $ghash_t1.16b                        // GHASH block 4k+1 - low
    aese    $ctr0.16b, $rk8.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 4k+4 - round 8
    PLATFORM_EOR3($acc_l, $acc_l, $ghash_t2, $ghash_t0)                       // GHASH block 4k/1/2/3 - low
    aese    $ctr1.16b, $rk8.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 4k+5 - round 8
    PLATFORM_EOR3($acc_m, $acc_m, $acc_h, $acc_l)                             // MODULO - karatsuba tidy up
    aese    $ctr2.16b, $rk8.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 4k+6 - round 8
    pmull    $ghash_t0.1q, $acc_h.1d, $mod_constant.1d                        // MODULO - top 64b align with mid
    aese    $ctr3.16b, $rk8.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 4k+7 - round 8
    cmp    $rounds, #12                                                       // setup flags for AES-128/192/256 check
    b.lt    .Lenc_main_loop_continue                                          // branch if AES-128
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #144]                               // load rk9, rk10
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 4k+4 - round 9
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 4k+5 - round 9
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 4k+6 - round 9
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 4k+7 - round 9
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 4k+4 - round 10
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 4k+5 - round 10
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 4k+6 - round 10
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 4k+7 - round 10
    b.eq    .Lenc_main_loop_continue                                          // branch if AES-192
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #176]                               // load rk11, rk12
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 4k+4 - round 11
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 4k+5 - round 11
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 4k+6 - round 11
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 4k+7 - round 11
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 4k+4 - round 12
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 4k+5 - round 12
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 4k+6 - round 12
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 4k+7 - round 12
.Lenc_main_loop_continue:
    ext    $acc_h.16b, $acc_h.16b, $acc_h.16b, #8                             // MODULO - other top alignment
    PLATFORM_EOR3($acc_m, $acc_m, $ghash_t0, $acc_h)                          // MODULO - fold into mid
    pmull    $acc_h.1q, $acc_m.1d, $mod_constant.1d                           // MODULO - mid 64b align with low
    ext    $ghash_t0.16b, $acc_m.16b, $acc_m.16b, #8                          // MODULO - other mid alignment
    PLATFORM_EOR3($acc_l, $acc_h, $acc_l, $ghash_t0)                          // MODULO - fold into low
    aese    $ctr0.16b, $rkNm1.16b                                             // AES block 4k+4 - round N-1
    PLATFORM_EOR3($res0, $res0, $v_rkN, $ctr0)                                // AES block 4k+4 - result
    aese    $ctr1.16b, $rkNm1.16b                                             // AES block 4k+5 - round N-1
    PLATFORM_EOR3($res1, $res1, $v_rkN, $ctr1)                                // AES block 4k+5 - result
    aese    $ctr2.16b, $rkNm1.16b                                             // AES block 4k+6 - round N-1
    PLATFORM_EOR3($res2, $res2, $v_rkN, $ctr2)                                // AES block 4k+6 - result
    aese    $ctr3.16b, $rkNm1.16b                                             // AES block 4k+7 - round N-1
    PLATFORM_EOR3($res3, $res3, $v_rkN, $ctr3)                                // AES block 4k+7 - result
    ldp    $ctr0q, $ctr1q, [sp, #$ctr0_sp_offset]
    ldp    $ctr2q, $ctr3q, [sp, #$ctr2_sp_offset]
    // We used these registers as temporaries above so reload the RKs.
    ldp    $rk2q, $rk3q, [$cc, #32]                                           // load rk2, rk3
    ldp    $rk4q, $rk5q, [$cc, #64]                                           // load rk4, rk5
    st1    { $res0.16b, $res1.16b, $res2.16b, $res3.16b}, [$output_ptr], #64  // AES blocks 4k+4-7 - store result
    rev    $tmp_gpr_w, $W_T32
    str    $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $W_T32, #1
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $W_T32, #2
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $W_T32, #3
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}]
    add    $W_T32, $W_T32, #4
    cmp    $input_ptr, $main_end_input_ptr                                    // .LOOP CONTROL
    b.lt    .Lenc_main_loop
.Lenc_prepretail:                                                             //    PREPRETAIL
    aese    $ctr1.16b, $rk0.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 0
    rev64    $res2.16b, $res2.16b                                             // GHASH block 2
    aese    $ctr2.16b, $rk0.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 0
    aese    $ctr0.16b, $rk0.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 0
    rev64    $res0.16b, $res0.16b                                             // GHASH block 0
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8                             // PRE 0
    aese    $ctr2.16b, $rk1.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 1
    aese    $ctr0.16b, $rk1.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 1
    eor    $res0.16b, $res0.16b, $acc_l.16b                                   // PRE 1
    rev64    $res1.16b, $res1.16b                                             // GHASH block 1
    aese    $ctr2.16b, $rk2.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 2
    aese    $ctr3.16b, $rk0.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 0
    mov    $acc_md, $h34k.d[1]                                                // GHASH block 0 - mid Karatsuba key
    aese    $ctr1.16b, $rk1.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 1
    pmull    $acc_l.1q, $res0.1d, $h4.1d                                      // GHASH block 0 - low
    mov    $mod_constantd, $res0.d[1]                                         // GHASH block 0 - mid
    pmull2    $acc_h.1q, $res0.2d, $h4.2d                                     // GHASH block 0 - high
    aese    $ctr2.16b, $rk3.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 3
    aese    $ctr1.16b, $rk2.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 2
    eor    $mod_constant.8b, $mod_constant.8b, $res0.8b                       // GHASH block 0 - mid
    aese    $ctr0.16b, $rk2.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 2
    aese    $ctr3.16b, $rk1.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 1
    aese    $ctr1.16b, $rk3.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 3
    pmull    $acc_m.1q, $mod_constant.1d, $acc_m.1d                           // GHASH block 0 - mid
    pmull2    $res0.1q, $res1.2d, $h3.2d                                      // GHASH block 1 - high
    pmull    $mod_constant.1q, $res1.1d, $h3.1d                               // GHASH block 1 - low
    aese    $ctr3.16b, $rk2.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 2
    eor    $acc_h.16b, $acc_h.16b, $res0.16b                                  // GHASH block 1 - high
    mov    $res0d, $res1.d[1]                                                 // GHASH block 1 - mid
    aese    $ctr0.16b, $rk3.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 3
    eor    $acc_l.16b, $acc_l.16b, $mod_constant.16b                          // GHASH block 1 - low
    aese    $ctr3.16b, $rk3.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 3
    eor    $res0.8b, $res0.8b, $res1.8b                                       // GHASH block 1 - mid
    mov    $mod_constantd, $res2.d[1]                                         // GHASH block 2 - mid
    aese    $ctr0.16b, $rk4.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 4
    rev64    $res3.16b, $res3.16b                                             // GHASH block 3
    aese    $ctr3.16b, $rk4.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 4
    pmull    $res0.1q, $res0.1d, $h34k.1d                                     // GHASH block 1 - mid
    eor    $mod_constant.8b, $mod_constant.8b, $res2.8b                       // GHASH block 2 - mid
    pmull    $res1.1q, $res2.1d, $h2.1d                                       // GHASH block 2 - low
    aese    $ctr3.16b, $rk5.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 5
    aese    $ctr2.16b, $rk4.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 4
    eor    $acc_m.16b, $acc_m.16b, $res0.16b                                  // GHASH block 1 - mid
    pmull2    $res0.1q, $res2.2d, $h2.2d                                      // GHASH block 2 - high
    eor    $acc_l.16b, $acc_l.16b, $res1.16b                                  // GHASH block 2 - low
    ins    $mod_constant.d[1], $mod_constant.d[0]                             // GHASH block 2 - mid
    aese    $ctr2.16b, $rk5.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 5
    eor    $acc_h.16b, $acc_h.16b, $res0.16b                                  // GHASH block 2 - high
    mov    $res0d, $res3.d[1]                                                 // GHASH block 3 - mid
    aese    $ctr1.16b, $rk4.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 4
    pmull2    $mod_constant.1q, $mod_constant.2d, $h12k.2d                    // GHASH block 2 - mid
    eor    $res0.8b, $res0.8b, $res3.8b                                       // GHASH block 3 - mid
    pmull2    $res1.1q, $res3.2d, $h1.2d                                      // GHASH block 3 - high
    aese    $ctr1.16b, $rk5.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 5
    pmull    $res0.1q, $res0.1d, $h12k.1d                                     // GHASH block 3 - mid
    eor    $acc_m.16b, $acc_m.16b, $mod_constant.16b                          // GHASH block 2 - mid
    aese    $ctr0.16b, $rk5.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 5
    aese    $ctr1.16b, $rk6.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 6
    aese    $ctr2.16b, $rk6.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 6
    aese    $ctr0.16b, $rk6.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 6
    aese    $ctr3.16b, $rk6.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 6
    aese    $ctr1.16b, $rk7.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 7
    eor    $acc_h.16b, $acc_h.16b, $res1.16b                                  // GHASH block 3 - high
    aese    $ctr0.16b, $rk7.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 7
    aese    $ctr3.16b, $rk7.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 7
    ldr    $mod_constantd, [sp, #$mod_constant_sp_offset]
    aese    $ctr1.16b, $rk8.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 8
    eor    $acc_m.16b, $acc_m.16b, $res0.16b                                  // GHASH block 3 - mid
    pmull    $res2.1q, $res3.1d, $h1.1d                                       // GHASH block 3 - low
    aese    $ctr3.16b, $rk8.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 8
    cmp    $rounds, #12                                                       // setup flags for AES-128/192/256 check
    aese    $ctr0.16b, $rk8.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 8
    eor    $acc_l.16b, $acc_l.16b, $res2.16b                                  // GHASH block 3 - low
    aese    $ctr2.16b, $rk7.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 7
    eor    $acc_m.16b, $acc_m.16b, $acc_h.16b                                 // karatsuba tidy up
    aese    $ctr2.16b, $rk8.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 8
    pmull    $res0.1q, $acc_h.1d, $mod_constant.1d
    ext    $acc_h.16b, $acc_h.16b, $acc_h.16b, #8
    eor    $acc_m.16b, $acc_m.16b, $acc_l.16b
    b.lt    .Lenc_finish_prepretail                                           // branch if AES-128
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #144]                               // load rk9, rk10
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 9
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 9
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 9
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 9
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 10
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 10
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 10
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 10
    b.eq    .Lenc_finish_prepretail                                           // branch if AES-192
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #176]                               // load rk11, rk12
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 11
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 11
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 11
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 11
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 12
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 12
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 12
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 12
.Lenc_finish_prepretail:
    aese    $ctr0.16b, $rkNm1.16b                                             // AES block 0 - round N-1
    aese    $ctr1.16b, $rkNm1.16b                                             // AES block 1 - round N-1
    aese    $ctr2.16b, $rkNm1.16b                                             // AES block 2 - round N-1
    aese    $ctr3.16b, $rkNm1.16b                                             // AES block 3 - round N-1
    PLATFORM_EOR3($acc_m, $acc_m, $res0, $acc_h)
    pmull    $res0.1q, $acc_m.1d, $mod_constant.1d
    ext    $acc_m.16b, $acc_m.16b, $acc_m.16b, #8
    PLATFORM_EOR3($acc_l, $acc_l, $res0, $acc_m)
.Lenc_tail:                                                                   // TAIL: Process remaining 0 to 3 blocks
    ext    $mod_constant.16b, $acc_l.16b, $acc_l.16b, #8                      // Save current GHASH state for partial tag feed-in
    sub    $main_end_input_ptr, $end_input_ptr, $input_ptr                    // main_end_input_ptr is number of bytes left to process
    ldp    $input_l0, $input_h0, [$input_ptr], #16                            // AES block 0 - load plaintext
    eor    $input_l0, $input_l0, $rkN_l                                       // AES block 0 - round N low
    eor    $input_h0, $input_h0, $rkN_h                                       // AES block 0 - round N high
    cmp    $main_end_input_ptr, #48
    fmov    $res0d, $input_l0                                                 // AES block 0 - mov low
    fmov    $res0.d[1], $input_h0                                             // AES block 0 - mov high
    eor    $res1.16b, $res0.16b, $ctr0.16b                                    // AES block 0 - result
    b.gt    .Lenc_blocks_more_than_3
    cmp    $main_end_input_ptr, #32
    mov    $ctr3.16b, $ctr2.16b
    movi    $acc_l.8b, #0
    movi    $acc_h.8b, #0
    mov    $ctr2.16b, $ctr1.16b
    movi    $acc_m.8b, #0
    b.gt    .Lenc_blocks_more_than_2
    mov    $ctr3.16b, $ctr1.16b
    cmp    $main_end_input_ptr, #16
    b.gt    .Lenc_blocks_more_than_1
    b    .Lenc_blocks_less_than_1
.Lenc_blocks_more_than_3:                                                     // blocks left >  3
    st1    { $res1.16b}, [$output_ptr], #16                                   // AES final-2 block - store result
    ldp    $input_l0, $input_h0, [$input_ptr], #16                            // AES final-2 block - load input low & high
    rev64    $res0.16b, $res1.16b                                             // GHASH final-3 block
    eor    $input_l0, $input_l0, $rkN_l                                       // AES final-2 block - round N low
    eor    $res0.16b, $res0.16b, $mod_constant.16b                            // feed in partial tag
    eor    $input_h0, $input_h0, $rkN_h                                       // AES final-2 block - round N high
    mov    $ghash_t2d, $res0.d[1]                                             // GHASH final-3 block - mid
    fmov    $res1d, $input_l0                                                 // AES final-2 block - mov low
    fmov    $res1.d[1], $input_h0                                             // AES final-2 block - mov high
    eor    $ghash_t2.8b, $ghash_t2.8b, $res0.8b                               // GHASH final-3 block - mid
    movi    $mod_constant.8b, #0                                              // suppress further partial tag feed in
    mov    $acc_md, $h34k.d[1]                                                // GHASH final-3 block - mid
    pmull    $acc_l.1q, $res0.1d, $h4.1d                                      // GHASH final-3 block - low
    pmull2    $acc_h.1q, $res0.2d, $h4.2d                                     // GHASH final-3 block - high
    pmull    $acc_m.1q, $ghash_t2.1d, $acc_m.1d                               // GHASH final-3 block - mid
    eor    $res1.16b, $res1.16b, $ctr1.16b                                    // AES final-2 block - result
.Lenc_blocks_more_than_2:                                                     // blocks left >  2
    st1    { $res1.16b}, [$output_ptr], #16                                   // AES final-2 block - store result
    ldp    $input_l0, $input_h0, [$input_ptr], #16                            // AES final-1 block - load input low & high
    rev64    $res0.16b, $res1.16b                                             // GHASH final-2 block
    eor    $input_l0, $input_l0, $rkN_l                                       // AES final-1 block - round N low
    eor    $res0.16b, $res0.16b, $mod_constant.16b                            // feed in partial tag
    fmov    $res1d, $input_l0                                                 // AES final-1 block - mov low
    eor    $input_h0, $input_h0, $rkN_h                                       // AES final-1 block - round N high
    fmov    $res1.d[1], $input_h0                                             // AES final-1 block - mov high
    movi    $mod_constant.8b, #0                                              // suppress further partial tag feed in
    pmull2    $ghash_t0.1q, $res0.2d, $h3.2d                                  // GHASH final-2 block - high
    mov    $ghash_t2d, $res0.d[1]                                             // GHASH final-2 block - mid
    pmull    $ghash_t1.1q, $res0.1d, $h3.1d                                   // GHASH final-2 block - low
    eor    $ghash_t2.8b, $ghash_t2.8b, $res0.8b                               // GHASH final-2 block - mid
    eor    $res1.16b, $res1.16b, $ctr2.16b                                    // AES final-1 block - result
    eor    $acc_h.16b, $acc_h.16b, $ghash_t0.16b                              // GHASH final-2 block - high
    pmull    $ghash_t2.1q, $ghash_t2.1d, $h34k.1d                             // GHASH final-2 block - mid
    eor    $acc_l.16b, $acc_l.16b, $ghash_t1.16b                              // GHASH final-2 block - low
    eor    $acc_m.16b, $acc_m.16b, $ghash_t2.16b                              // GHASH final-2 block - mid
.Lenc_blocks_more_than_1:                                                     // blocks left >  1
    st1    { $res1.16b}, [$output_ptr], #16                                   // AES final-1 block - store result
    rev64    $res0.16b, $res1.16b                                             // GHASH final-1 block: Byte Swap CT
    ldp    $input_l0, $input_h0, [$input_ptr], #16                            // AES final block - load plaintext
    eor    $res0.16b, $res0.16b, $mod_constant.16b                            // Feed in partial tag
    movi    $mod_constant.8b, #0                                              // Clear for next block
    eor    $input_l0, $input_l0, $rkN_l                                       // AES final block - round N low
    mov    $ghash_t2d, $res0.d[1]                                             // GHASH final-1 block - mid
    pmull2    $ghash_t0.1q, $res0.2d, $h2.2d                                  // GHASH final-1 block - high
    eor    $input_h0, $input_h0, $rkN_h                                       // AES final block - round N high
    eor    $ghash_t2.8b, $ghash_t2.8b, $res0.8b                               // GHASH final-1 block - mid
    eor    $acc_h.16b, $acc_h.16b, $ghash_t0.16b                              // GHASH final-1 block - high
    ins    $ghash_t2.d[1], $ghash_t2.d[0]                                     // GHASH final-1 block - mid
    fmov    $res1d, $input_l0                                                 // AES final block - mov low
    fmov    $res1.d[1], $input_h0                                             // AES final block - mov high
    pmull2    $ghash_t2.1q, $ghash_t2.2d, $h12k.2d                            // GHASH final-1 block - mid
    pmull    $ghash_t1.1q, $res0.1d, $h2.1d                                   // GHASH final-1 block - low
    eor    $res1.16b, $res1.16b, $ctr3.16b                                    // AES final block - result
    eor    $acc_m.16b, $acc_m.16b, $ghash_t2.16b                              // GHASH final-1 block - mid
    eor    $acc_l.16b, $acc_l.16b, $ghash_t1.16b                              // GHASH final-1 block - low
.Lenc_blocks_less_than_1:                                                     // Last partial block handling
    add    $T32, $T32, $bit_length, lsr #7                                    // Calculate the updated counter based on the number of 16B chunks we processed
    rev    $W32, $W32
    str    $W32, [$counter, #12]                                              // store the updated counter
    and    $bit_length, $bit_length, #127                                     // bit_length %= 128
    mvn    $rkN_l, xzr                                                        // Mask for low 64 bits
    sub    $bit_length, $bit_length, #128                                     //
    neg    $bit_length, $bit_length                                           // Valid bits in the last block (1-128)
    ldr    $final_block_destq, [$output_ptr]                                  // Load destination for merging
    mvn    $rkN_h, xzr                                                        // Mask for high 64 bits
    and    $bit_length, $bit_length, #127                                     // bit_length %= 128
    lsr    $rkN_h, $rkN_h, $bit_length                                        // rkN_h is mask for top 64b of last block
    cmp    $bit_length, #64
    csel    $input_l0, $rkN_l, $rkN_h, lt
    csel    $input_h0, $rkN_h, xzr, lt
    fmov    $ctr0d, $input_l0                                                 // ctr0d is mask for last block
    fmov    $ctr0.d[1], $input_h0
    and    $res1.16b, $res1.16b, $ctr0.16b                                    // Mask out unused bits of the last CT block
    rev64    $res0.16b, $res1.16b                                             // GHASH final block - byte swap
    eor    $res0.16b, $res0.16b, $mod_constant.16b                            // Feed in partial tag
    bif    $res1.16b, $final_block_dest.16b, $ctr0.16b                        // Bitwise Insert: merge with existing data at output_ptr
    pmull2    $ghash_t0.1q, $res0.2d, $h1.2d                                  // GHASH final block - high
    mov    $mod_constantd, $res0.d[1]                                         // GHASH final block - mid
    pmull    $ghash_t1.1q, $res0.1d, $h1.1d                                   // GHASH final block - low
    eor    $acc_h.16b, $acc_h.16b, $ghash_t0.16b                              // GHASH final block - high
    eor    $mod_constant.8b, $mod_constant.8b, $res0.8b                       // GHASH final block - mid
    pmull    $mod_constant.1q, $mod_constant.1d, $h12k.1d                     // GHASH final block - mid
    eor    $acc_l.16b, $acc_l.16b, $ghash_t1.16b                              // GHASH final block - low
    eor    $acc_m.16b, $acc_m.16b, $mod_constant.16b                          // GHASH final block - mid
    eor    $res0.16b, $acc_l.16b, $acc_h.16b                                  // MODULO - karatsuba tidy up
    fmov    $mod_constantd, $mod_constantx
    eor    $acc_m.16b, $acc_m.16b, $res0.16b                                  // MODULO - karatsuba tidy up
    pmull    $res3.1q, $acc_h.1d, $mod_constant.1d                            // MODULO - top 64b align with mid
    ext    $acc_h.16b, $acc_h.16b, $acc_h.16b, #8                             // MODULO - other top alignment
    PLATFORM_EOR3($acc_m, $acc_m, $res3, $acc_h)                              // MODULO - fold into mid
    pmull    $acc_h.1q, $acc_m.1d, $mod_constant.1d                           // MODULO - mid 64b align with low
    ext    $acc_m.16b, $acc_m.16b, $acc_m.16b, #8                             // MODULO - other mid alignment
    st1    { $res1.16b}, [$output_ptr]                                        // store all 16B
    PLATFORM_EOR3($acc_l, $acc_l, $acc_h, $acc_m)                             // MODULO - fold into low
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8                             // Byte swap GHASH result
    rev64    $acc_l.16b, $acc_l.16b                                           // Final Tag
    mov    $input_ptr, $len
    st1    { $acc_l.16b }, [$current_tag]                                     // Store final tag
    ldp    x19, x20, [sp, #16]
    ldp    $mod_constantx, x22, [sp, #32]
    ldp    d8, d9, [sp, #64]
    ldp    d10, d11, [sp, #80]
    ldp    d12, d13, [sp, #96]
    ldp    d14, d15, [sp, #112]
    ldp    x29, x30, [sp], #224
    AARCH64_VALIDATE_LINK_REGISTER
    ret
.size aes_gcm_enc_kernel,.-aes_gcm_enc_kernel
___
    ################################################################################
    # aes_gcm_dec_kernel
    ################################################################################
    $code_template .= <<"___";
.global aes_gcm_dec_kernel
.type   aes_gcm_dec_kernel,%function
.align  4
aes_gcm_dec_kernel:
    AARCH64_SIGN_LINK_REGISTER
    stp    x29, x30, [sp, #-224]!
    mov    x29, sp
    stp    x19, x20, [sp, #16]
    ld1    { $ctr0.16b}, [x4]
    mov    $ctr1.16b, $ctr0.16b
    mov    $ctr2.16b, $ctr0.16b
    mov    $ctr3.16b, $ctr0.16b
    mov    $counter, x4
    mov    $cc, x5
    stp    $mod_constantx, x22, [sp, #32]
    stp    x23, x24, [sp, #48]
    stp    d8, d9, [sp, #64]
    stp    d10, d11, [sp, #80]
    stp    d12, d13, [sp, #96]
    stp    d14, d15, [sp, #112]
    ldr    $rounds_w, [$cc, #240]                                             // Load number of AES rounds
    add    $input_l1, $cc, $rounds, lsl #4                                    // borrow input_l1 for last key
    ldp    $rkN_l, $rkN_h, [$input_l1]                                        // load round N keys
    ldr    $rkNm1q, [$input_l1, #-16]                                         // load round N-1 keys
    add    $end_input_ptr, $input_ptr, $bit_length, lsr #3                    // end_input_ptr
    lsr    $main_end_input_ptr, $bit_length, #3                               // byte_len
    mov    $len, $main_end_input_ptr
    ldr    $ctr32w, [$counter, #12]                                           // Load scalar 32-bit counter (CTR)
    sub    $main_end_input_ptr, $main_end_input_ptr, #1                       // byte_len - 1
    ldr    $rk0q, [$cc, #0]                                                   // load rk0
    and    $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0      // number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
    add    $main_end_input_ptr, $main_end_input_ptr, $input_ptr
    rev    $ctr32w, $ctr32w                                                   // Reverse it once for big-endian incrementing
    uxtw   $T32, $ctr32w                                                      // Zero extend reversed w9 into x10
    str    $ctr0q,     [sp, #$ctr0_sp_offset]
    str    $ctr0q,     [sp, #$ctr1_sp_offset]
    str    $ctr0q,     [sp, #$ctr2_sp_offset]
    str    $ctr0q,     [sp, #$ctr3_sp_offset]
    rev    $tmp_gpr_w, $ctr32w
    mov    $ctr0.s[3], $tmp_gpr_w
    add    $tmp_gpr_w, $ctr32w, #1
    rev    $tmp_gpr_w, $tmp_gpr_w
    mov    $ctr1.s[3], $tmp_gpr_w
    add    $tmp_gpr_w, $ctr32w, #2
    rev    $tmp_gpr_w, $tmp_gpr_w
    mov    $ctr2.s[3], $tmp_gpr_w
    add    $tmp_gpr_w, $ctr32w, #3
    rev    $tmp_gpr_w, $tmp_gpr_w
    mov    $ctr3.s[3], $tmp_gpr_w
    add    $tmp_gpr_w, $ctr32w, #4
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #5
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #6
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #7
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}]
    add    $ctr32w, $ctr32w, #8
    // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop
    mov    $mod_constantx, #0xc200000000000000
    str    $mod_constantx, [sp, #$mod_constant_sp_offset]
    aese    $ctr0.16b, $rk0.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 0
    ldp    $rk1q, $rk2q, [$cc, #16]                                           // load rk1, rk2
    aese    $ctr1.16b, $rk0.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 0
    ldp    $rk3q, $rk4q, [$cc, #48]                                           // load rk3, rk4
    aese    $ctr2.16b, $rk0.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 0
    ldp    $rk5q, $rk6q, [$cc, #80]                                           // load rk5, rk6
    aese    $ctr3.16b, $rk0.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 0
    ldp    $h2q, $h3q, [$Htable, #32]                                         // load h2, h3
    aese    $ctr0.16b, $rk1.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 1
    ldp    $rk7q, $rk8q, [$cc, #112]                                          // load rk7, rk8
    aese    $ctr1.16b, $rk1.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 1
    aese    $ctr2.16b, $rk1.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 1
    aese    $ctr3.16b, $rk1.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 1
    aese    $ctr0.16b, $rk2.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 2
    ext    $h3.16b, $h3.16b, $h3.16b, #8
    aese    $ctr1.16b, $rk2.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 2
    ext    $h2.16b, $h2.16b, $h2.16b, #8
    ldr    $h4q, [$Htable, #80]                                               // load h4
    aese    $ctr2.16b, $rk2.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 2
    aese    $ctr3.16b, $rk2.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 2
    ext    $h4.16b, $h4.16b, $h4.16b, #8
    aese    $ctr0.16b, $rk3.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 3
    ld1    { $acc_l.16b}, [$current_tag]
    aese    $ctr1.16b, $rk3.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 3
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8
    aese    $ctr2.16b, $rk3.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 3
    rev64    $acc_l.16b, $acc_l.16b
    aese    $ctr3.16b, $rk3.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 3
    trn2    $h34k.2d,  $h3.2d,    $h4.2d                                      // h4l | h3l
    aese    $ctr0.16b, $rk4.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 4
    ldr    $h1q, [$Htable]                                                    // load h1
    aese    $ctr1.16b, $rk4.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 4
    ext    $h1.16b, $h1.16b, $h1.16b, #8
    aese    $ctr2.16b, $rk4.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 4
    trn1    $acc_h.2d, $h3.2d,    $h4.2d                                      // h4h | h3h
    aese    $ctr3.16b, $rk4.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 4
    trn2    $h12k.2d,  $h1.2d,    $h2.2d                                      // h2l | h1l
    aese    $ctr0.16b, $rk5.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 5
    aese    $ctr1.16b, $rk5.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 5
    aese    $ctr3.16b, $rk5.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 5
    aese    $ctr2.16b, $rk5.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 5
    aese    $ctr0.16b, $rk6.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 6
    aese    $ctr1.16b, $rk6.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 6
    aese    $ctr2.16b, $rk6.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 6
    aese    $ctr3.16b, $rk6.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 6
    aese    $ctr0.16b, $rk7.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 7
    aese    $ctr1.16b, $rk7.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 7
    aese    $ctr2.16b, $rk7.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 7
    aese    $ctr3.16b, $rk7.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 7
    aese    $ctr0.16b, $rk8.16b \n    aesmc    $ctr0.16b, $ctr0.16b           // AES block 0 - round 8
    aese    $ctr1.16b, $rk8.16b \n    aesmc    $ctr1.16b, $ctr1.16b           // AES block 1 - round 8
    aese    $ctr2.16b, $rk8.16b \n    aesmc    $ctr2.16b, $ctr2.16b           // AES block 2 - round 8
    aese    $ctr3.16b, $rk8.16b \n    aesmc    $ctr3.16b, $ctr3.16b           // AES block 3 - round 8
    cmp    $rounds, #12                                                       // setup flags for AES-128/192/256 check
    b.lt    .Ldec_finish_first_blocks                                         // branch if AES-128
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #144]                               // load rk9, rk10
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 9
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 9
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 9
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 9
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 10
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 10
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 10
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 10
    b.eq    .Ldec_finish_first_blocks                                         // branch if AES-192
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #176]                               // load rk11, rk12
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 11
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 11
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 11
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 11
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 12
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 12
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 12
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 12
.Ldec_finish_first_blocks:
    ldr    $rk9_11_tmpq, [$input_l1]                                          // load rkN
    cmp    $input_ptr, $main_end_input_ptr                                    // check if we have <= 4 blocks
    eor    $h34k.16b, $h34k.16b, $acc_h.16b                                   // h4k | h3k
    aese    $ctr1.16b, $rkNm1.16b                                             // AES block 1 - round N-1
    aese    $ctr2.16b, $rkNm1.16b                                             // AES block 2 - round N-1
    aese    $ctr3.16b, $rkNm1.16b                                             // AES block 3 - round N-1
    aese    $ctr0.16b, $rkNm1.16b                                             // AES block 0 - round N-1
    trn1    $t0.2d,    $h1.2d,    $h2.2d                                      // h2h | h1h
    eor    $h12k.16b, $h12k.16b, $t0.16b                                      // h2k | h1k
    b.ge    .Ldec_tail                                                        // handle tail
    // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions.
    // This is because the final result of the AES block needs to be EORd with the final round key
    // value ($v_rkN). This avoids several fmovs.
    ldp    $res2q, $res3q, [$input_ptr, #32]                                  // AES blocks 2,3 load ciphertext
    ldp    $res0q, $res1q, [$input_ptr], #64                                  // AES blocks 0,1 load ciphertext
    PLATFORM_EOR3($ctr0, $res0, $ctr0, $rk9_11_tmp)                           // AES block 0 - result
    PLATFORM_EOR3($ctr1, $res1, $ctr1, $rk9_11_tmp)                           // AES block 1 - result
    PLATFORM_EOR3($ctr2, $res2, $ctr2, $rk9_11_tmp)                           // AES block 2 - result
    PLATFORM_EOR3($ctr3, $res3, $ctr3, $rk9_11_tmp)                           // AES block 3 - result
    st1    { $ctr0.16b, $ctr1.16b, $ctr2.16b, $ctr3.16b}, [$output_ptr], #64  // AES blocks 0-3 - store result
    ldr    $ctr0q,     [sp, #$ctr0_sp_offset]
    ldr    $ctr1q,     [sp, #$ctr1_sp_offset]
    ldr    $ctr2q,     [sp, #$ctr2_sp_offset]
    ldr    $ctr3q,     [sp, #$ctr3_sp_offset]
    rev    $tmp_gpr_w, $ctr32w
    str    $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #1
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #2
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #3
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}]
    add    $ctr32w, $ctr32w, #4
    cmp    $input_ptr, $main_end_input_ptr                                    // check if we have <= 4 blocks
    b.ge    .Ldec_prepretail                                                  // do prepretail
.Ldec_main_loop:    // main loop start
    aese    $ctr0.16b, $rk0.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 0
    aese    $ctr1.16b, $rk0.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 0
    aese    $ctr2.16b, $rk0.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 0
    aese    $ctr3.16b, $rk0.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 0
    aese    $ctr0.16b, $rk1.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 1
    aese    $ctr1.16b, $rk1.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 1
    aese    $ctr2.16b, $rk1.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 1
    aese    $ctr3.16b, $rk1.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 1
    rev64    $res0.16b, $res0.16b                                             // GHASH block 4k
    rev64    $res1.16b, $res1.16b                                             // GHASH block 4k+1
    rev64    $res2.16b, $res2.16b                                             // GHASH block 4k+2
    rev64    $res3.16b, $res3.16b                                             // GHASH block 4k+3
    aese    $ctr0.16b, $rk2.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 2
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8                             // PRE 0
    aese    $ctr1.16b, $rk2.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 2
    eor    $res0.16b, $res0.16b, $acc_l.16b                                   // PRE 1
    aese    $ctr2.16b, $rk2.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 2
    pmull    $acc_l.1q, $res0.1d, $h4.1d                                      // GHASH block 4k - low
    aese    $ctr3.16b, $rk2.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 2
    pmull2    $acc_h.1q, $res0.2d, $h4.2d                                     // GHASH block 4k - high
    aese    $ctr0.16b, $rk3.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 3
    mov    $acc_md, $h34k.d[1]                                                // GHASH block 4k - mid
    aese    $ctr1.16b, $rk3.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 3
    mov    $t0d, $res0.d[1]                                                   // GHASH block 4k - mid
    aese    $ctr2.16b, $rk3.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 3
    eor    $t0.8b, $t0.8b, $res0.8b                                           // GHASH block 4k - mid
    aese    $ctr3.16b, $rk3.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 3
    aese    $ctr0.16b, $rk4.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 4
    pmull2    $t1.1q, $res1.2d, $h3.2d                                        // GHASH block 4k+1 - high
    aese    $ctr1.16b, $rk4.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 4
    eor    $acc_h.16b, $acc_h.16b, $t1.16b                                    // GHASH block 4k+1 - high
    aese    $ctr2.16b, $rk4.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 4
    mov    $t3d, $res1.d[1]                                                   // GHASH block 4k+1 - mid
    aese    $ctr3.16b, $rk4.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 4
    pmull    $acc_m.1q, $t0.1d, $acc_m.1d                                     // GHASH block 4k - mid
    aese    $ctr0.16b, $rk5.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 5
    pmull    $t2.1q, $res1.1d, $h3.1d                                         // GHASH block 4k+1 - low
    aese    $ctr1.16b, $rk5.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 5
    eor    $t3.8b, $t3.8b, $res1.8b                                           // GHASH block 4k+1 - mid
    aese    $ctr3.16b, $rk5.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 5
    aese    $ctr2.16b, $rk5.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 5
    pmull    $t5.1q, $res2.1d, $h2.1d                                         // GHASH block 4k+2 - low
    aese    $ctr0.16b, $rk6.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 6
    PLATFORM_EOR3($acc_l, $acc_l, $t2, $t5)                                   // GHASH block 4k+1 - low & GHASH block 4k+2 - low
    aese    $ctr1.16b, $rk6.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 6
    mov    $t6d, $res2.d[1]                                                   // GHASH block 4k+2 - mid
    aese    $ctr2.16b, $rk6.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 6
    eor    $t6.8b, $t6.8b, $res2.8b                                           // GHASH block 4k+2 - mid
    pmull    $t3.1q, $t3.1d, $h34k.1d                                         // GHASH block 4k+1 - mid
    ins    $t6.d[1], $t6.d[0]                                                 // GHASH block 4k+2 - mid
    aese    $ctr3.16b, $rk6.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 6
    pmull2    $rk10_12.1q, $res2.2d, $h2.2d                                   // GHASH block 4k+2 - high
    mov    $t9d, $res3.d[1]                                                   // GHASH block 4k+3 - mid
    aese    $ctr0.16b, $rk7.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 7
    pmull2    $t6.1q, $t6.2d, $h12k.2d                                        // GHASH block 4k+2 - mid
    aese    $ctr1.16b, $rk7.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 7
    pmull    $rk9_11_tmp.1q, $res3.1d, $h1.1d                                 // GHASH block 4k+3 - low
    aese    $ctr2.16b, $rk7.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 7
    PLATFORM_EOR3($acc_m, $acc_m, $t3, $t6)                                   // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid
    aese    $ctr3.16b, $rk7.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 7
    pmull2    $t7.1q, $res3.2d, $h1.2d                                        // GHASH block 4k+3 - high
    eor    $t9.8b, $t9.8b, $res3.8b                                           // GHASH block 4k+3 - mid
    PLATFORM_EOR3($acc_h, $acc_h, $rk10_12, $t7)                              // GHASH block 4k+2 - high & GHASH block 4k+3 - high
    aese    $ctr0.16b, $rk8.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 4k+4 - round 8
    aese    $ctr1.16b, $rk8.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 4k+5 - round 8
    aese    $ctr2.16b, $rk8.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 4k+6 - round 8
    pmull    $t9.1q, $t9.1d, $h12k.1d                                         // GHASH block 4k+3 - mid
    ldr    $mod_constantd, [sp, #$mod_constant_sp_offset]
    aese    $ctr3.16b, $rk8.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 4k+7 - round 8
    pmull    $mod_t.1q, $acc_h.1d, $mod_constant.1d                           // MODULO - top 64b align with mid
    PLATFORM_EOR3($acc_m, $acc_m, $t9, $mod_t)                                // GHASH block 4k+3 - mid & MODULO - fold into mid
    eor    $acc_l.16b, $acc_l.16b, $rk9_11_tmp.16b                            // GHASH block 4k+3 - low
    eor    $t9.16b, $acc_l.16b, $acc_h.16b                                    // MODULO - karatsuba tidy up
    ext    $acc_h.16b, $acc_h.16b, $acc_h.16b, #8                             // MODULO - other top alignment
    PLATFORM_EOR3($acc_m, $acc_m, $t9, $acc_h)                                // MODULO - karatsuba tidy up & MODULO - fold into mid
    pmull    $mod_constant.1q, $acc_m.1d, $mod_constant.1d                    // MODULO - mid 64b align with low
    ext    $acc_m.16b, $acc_m.16b, $acc_m.16b, #8                             // MODULO - other mid alignment
    PLATFORM_EOR3($acc_l, $acc_l, $mod_constant, $acc_m)                      // MODULO - fold into low
    cmp    $rounds_w, #12                                                     // setup flags for AES-128/192/256 check
    b.lt    .Ldec_main_loop_continue                                          // branch if AES-128
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #144]                               // load rk9, rk10
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 4k+4 - round 9
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 4k+5 - round 9
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 4k+6 - round 9
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 4k+7 - round 9
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 4k+4 - round 10
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 4k+5 - round 10
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 4k+6 - round 10
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 4k+7 - round 10
    b.eq    .Ldec_main_loop_continue                                          // branch if AES-192
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #176]                               // load rk11, rk12
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 4k+4 - round 11
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 4k+5 - round 11
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 4k+6 - round 11
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 4k+7 - round 11
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 4k+4 - round 12
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 4k+5 - round 12
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 4k+6 - round 12
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 4k+7 - round 12
.Ldec_main_loop_continue:
    ldr    $rk9_11_tmpq, [$input_l1]                                          // load rkN
    ldp    $res2q, $res3q, [$input_ptr, #32]                                  // AES blocks 2,3 load ciphertext
    ldp    $res0q, $res1q, [$input_ptr], #64                                  // AES blocks 0,1 load ciphertext
    aese    $ctr0.16b, $rkNm1.16b                                             // AES block 4k+4 - round N-1
    PLATFORM_EOR3($ctr0, $res0, $ctr0, $rk9_11_tmp)                           // AES block 4k+4 - result
    aese    $ctr1.16b, $rkNm1.16b                                             // AES block 4k+5 - round N-1
    PLATFORM_EOR3($ctr1, $res1, $ctr1, $rk9_11_tmp)                           // AES block 4k+5 - result
    aese    $ctr2.16b, $rkNm1.16b                                             // AES block 4k+6 - round N-1
    PLATFORM_EOR3($ctr2, $res2, $ctr2, $rk9_11_tmp)                           // AES block 4k+6 - result
    aese    $ctr3.16b, $rkNm1.16b                                             // AES block 4k+7 - round N-1
    PLATFORM_EOR3($ctr3, $res3, $ctr3, $rk9_11_tmp)                           // AES block 4k+7 - result
    st1    { $ctr0.16b, $ctr1.16b, $ctr2.16b, $ctr3.16b}, [$output_ptr], #64  // AES blocks 4k+4-7 - store result
    ldr    $ctr0q,     [sp, #$ctr0_sp_offset]
    ldr    $ctr1q,     [sp, #$ctr1_sp_offset]
    ldr    $ctr2q,     [sp, #$ctr2_sp_offset]
    ldr    $ctr3q,     [sp, #$ctr3_sp_offset]
    rev    $tmp_gpr_w, $ctr32w
    str    $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #1
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #2
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}]
    add    $tmp_gpr_w, $ctr32w, #3
    rev    $tmp_gpr_w, $tmp_gpr_w
    str    $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}]
    add    $ctr32w, $ctr32w, #4
    cmp    $input_ptr, $main_end_input_ptr
    b.lt    .Ldec_main_loop
.Ldec_prepretail:                                                             // PREPRETAIL
    rev64    $res0.16b, $res0.16b                                             // GHASH block 0
    rev64    $res1.16b, $res1.16b                                             // GHASH block 1
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8                             // PRE 0
    aese    $ctr0.16b, $rk0.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 0
    aese    $ctr1.16b, $rk0.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 0
    eor    $res0.16b, $res0.16b, $acc_l.16b                                   // PRE 1
    rev64    $res2.16b, $res2.16b                                             // GHASH block 2
    aese    $ctr1.16b, $rk1.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 1
    pmull    $acc_l.1q, $res0.1d, $h4.1d                                      // GHASH block 0 - low
    mov    $t0d, $res0.d[1]                                                   // GHASH block 0 - mid
    pmull2    $acc_h.1q, $res0.2d, $h4.2d                                     // GHASH block 0 - high
    aese    $ctr2.16b, $rk0.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 0
    mov    $acc_md, $h34k.d[1]                                                // GHASH block 0 - mid
    aese    $ctr0.16b, $rk1.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 1
    eor    $t0.8b, $t0.8b, $res0.8b                                           // GHASH block 0 - mid
    pmull2    $t1.1q, $res1.2d, $h3.2d                                        // GHASH block 1 - high
    aese    $ctr2.16b, $rk1.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 1
    rev64    $res3.16b, $res3.16b                                             // GHASH block 3
    aese    $ctr3.16b, $rk0.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 0
    pmull    $acc_m.1q, $t0.1d, $acc_m.1d                                     // GHASH block 0 - mid
    eor    $acc_h.16b, $acc_h.16b, $t1.16b                                    // GHASH block 1 - high
    pmull    $t2.1q, $res1.1d, $h3.1d                                         // GHASH block 1 - low
    aese    $ctr3.16b, $rk1.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 1
    mov    $t3d, $res1.d[1]                                                   // GHASH block 1 - mid
    aese    $ctr0.16b, $rk2.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 2
    aese    $ctr1.16b, $rk2.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 2
    eor    $acc_l.16b, $acc_l.16b, $t2.16b                                    // GHASH block 1 - low
    aese    $ctr2.16b, $rk2.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 2
    aese    $ctr0.16b, $rk3.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 3
    mov    $t6d, $res2.d[1]                                                   // GHASH block 2 - mid
    aese    $ctr3.16b, $rk2.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 2
    eor    $t3.8b, $t3.8b, $res1.8b                                           // GHASH block 1 - mid
    pmull    $t5.1q, $res2.1d, $h2.1d                                         // GHASH block 2 - low
    aese    $ctr0.16b, $rk4.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 4
    aese    $ctr3.16b, $rk3.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 3
    eor    $t6.8b, $t6.8b, $res2.8b                                           // GHASH block 2 - mid
    pmull    $t3.1q, $t3.1d, $h34k.1d                                         // GHASH block 1 - mid
    aese    $ctr0.16b, $rk5.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 5
    eor    $acc_l.16b, $acc_l.16b, $t5.16b                                    // GHASH block 2 - low
    aese    $ctr3.16b, $rk4.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 4
    pmull2    $t7.1q, $res3.2d, $h1.2d                                        // GHASH block 3 - high
    eor    $acc_m.16b, $acc_m.16b, $t3.16b                                    // GHASH block 1 - mid
    pmull2    $t4.1q, $res2.2d, $h2.2d                                        // GHASH block 2 - high
    aese    $ctr3.16b, $rk5.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 5
    ins    $t6.d[1], $t6.d[0]                                                 // GHASH block 2 - mid
    aese    $ctr2.16b, $rk3.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 3
    aese    $ctr1.16b, $rk3.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 3
    PLATFORM_EOR3($acc_h, $acc_h, $t4, $t7)                                   // GHASH block 2 - high & GHASH block 3 - high
    pmull    $t8.1q, $res3.1d, $h1.1d                                         // GHASH block 3 - low
    aese    $ctr2.16b, $rk4.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 4
    mov    $t9d, $res3.d[1]                                                   // GHASH block 3 - mid
    aese    $ctr1.16b, $rk4.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 4
    pmull2    $t6.1q, $t6.2d, $h12k.2d                                        // GHASH block 2 - mid
    aese    $ctr2.16b, $rk5.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 5
    eor    $t9.8b, $t9.8b, $res3.8b                                           // GHASH block 3 - mid
    aese    $ctr1.16b, $rk5.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 5
    aese    $ctr3.16b, $rk6.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 6
    eor    $acc_m.16b, $acc_m.16b, $t6.16b                                    // GHASH block 2 - mid
    aese    $ctr2.16b, $rk6.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 6
    aese    $ctr0.16b, $rk6.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 6
    movi    $mod_constant.8b, #0xc2
    aese    $ctr1.16b, $rk6.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 6
    eor    $acc_l.16b, $acc_l.16b, $t8.16b                                    // GHASH block 3 - low
    pmull    $t9.1q, $t9.1d, $h12k.1d                                         // GHASH block 3 - mid
    aese    $ctr3.16b, $rk7.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 7
    aese    $ctr1.16b, $rk7.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 7
    aese    $ctr0.16b, $rk7.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 7
    eor    $acc_m.16b, $acc_m.16b, $t9.16b                                    // GHASH block 3 - mid
    aese    $ctr3.16b, $rk8.16b  \n    aesmc    $ctr3.16b, $ctr3.16b          // AES block 3 - round 8
    aese    $ctr2.16b, $rk7.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 7
    eor    $t9.16b, $acc_l.16b, $acc_h.16b                                    // MODULO - karatsuba tidy up
    aese    $ctr1.16b, $rk8.16b  \n    aesmc    $ctr1.16b, $ctr1.16b          // AES block 1 - round 8
    aese    $ctr0.16b, $rk8.16b  \n    aesmc    $ctr0.16b, $ctr0.16b          // AES block 0 - round 8
    shl    $mod_constantd, $mod_constantd, #56                                // mod_constant
    aese    $ctr2.16b, $rk8.16b  \n    aesmc    $ctr2.16b, $ctr2.16b          // AES block 2 - round 8
    cmp    $rounds_w, #12                                                     // setup flags for AES-128/192/256 check
    b.lt    .Ldec_finish_prepretail                                           // branch if AES-128
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #144]                               // load rk9, rk10
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 9
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 9
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 9
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 9
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 10
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 10
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 10
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 10
    b.eq    .Ldec_finish_prepretail                                           // branch if AES-192
    ldp    $rk9_11_tmpq, $rk10_12q, [$cc, #176]                               // load rk11, rk12
    aese    $ctr0.16b, $rk9_11_tmp.16b \n    aesmc    $ctr0.16b, $ctr0.16b    // AES block 0 - round 11
    aese    $ctr1.16b, $rk9_11_tmp.16b \n    aesmc    $ctr1.16b, $ctr1.16b    // AES block 1 - round 11
    aese    $ctr2.16b, $rk9_11_tmp.16b \n    aesmc    $ctr2.16b, $ctr2.16b    // AES block 2 - round 11
    aese    $ctr3.16b, $rk9_11_tmp.16b \n    aesmc    $ctr3.16b, $ctr3.16b    // AES block 3 - round 11
    aese    $ctr0.16b, $rk10_12.16b \n    aesmc    $ctr0.16b, $ctr0.16b       // AES block 0 - round 12
    aese    $ctr1.16b, $rk10_12.16b \n    aesmc    $ctr1.16b, $ctr1.16b       // AES block 1 - round 12
    aese    $ctr2.16b, $rk10_12.16b \n    aesmc    $ctr2.16b, $ctr2.16b       // AES block 2 - round 12
    aese    $ctr3.16b, $rk10_12.16b \n    aesmc    $ctr3.16b, $ctr3.16b       // AES block 3 - round 12
.Ldec_finish_prepretail:
    eor    $acc_m.16b, $acc_m.16b, $t9.16b                                    // MODULO - karatsuba tidy up
    pmull    $mod_t.1q, $acc_h.1d, $mod_constant.1d                           // MODULO - top 64b align with mid
    ext    $acc_h.16b, $acc_h.16b, $acc_h.16b, #8                             // MODULO - other top alignment
    PLATFORM_EOR3($acc_m, $acc_m, $mod_t, $acc_h)                             // MODULO - fold into mid
    pmull    $mod_constant.1q, $acc_m.1d, $mod_constant.1d                    // MODULO - mid 64b align with low
    ext    $acc_m.16b, $acc_m.16b, $acc_m.16b, #8                             // MODULO - other mid alignment
    PLATFORM_EOR3($acc_l, $acc_l, $mod_constant, $acc_m)                      // MODULO - fold into low
    aese    $ctr1.16b, $rkNm1.16b                                             // AES block 1 - round N-1
    aese    $ctr0.16b, $rkNm1.16b                                             // AES block 0 - round N-1
    aese    $ctr3.16b, $rkNm1.16b                                             // AES block 3 - round N-1
    aese    $ctr2.16b, $rkNm1.16b                                             // AES block 2 - round N-1
.Ldec_tail:                                                                   // TAIL
    sub    $main_end_input_ptr, $end_input_ptr, $input_ptr                    // main_end_input_ptr is number of bytes left to process
    ld1    { $res1.16b}, [$input_ptr], #16                                    // AES block 0 - load ciphertext
    eor    $ctr0.16b, $res1.16b, $ctr0.16b                                    // AES block 0 - result
    mov    $output_l0, $ctr0.d[0]                                             // AES block 0 - mov low
    mov    $output_h0, $ctr0.d[1]                                             // AES block 0 - mov high
    ext    $t0.16b, $acc_l.16b, $acc_l.16b, #8                                // prepare final partial tag
    eor    $output_l0, $output_l0, $rkN_l                                     // AES block 0 - round N low
    eor    $output_h0, $output_h0, $rkN_h                                     // AES block 0 - round N high
    cmp    $main_end_input_ptr, #48
    b.gt    .Ldec_blocks_more_than_3
    mov    $ctr3.16b, $ctr2.16b
    movi    $acc_m.8b, #0
    movi    $acc_l.8b, #0
    movi    $acc_h.8b, #0
    mov    $ctr2.16b, $ctr1.16b
    cmp    $main_end_input_ptr, #32
    b.gt    .Ldec_blocks_more_than_2
    mov    $ctr3.16b, $ctr1.16b
    cmp    $main_end_input_ptr, #16
    b.gt    .Ldec_blocks_more_than_1
    b    .Ldec_blocks_less_than_1
.Ldec_blocks_more_than_3:                                                     // blocks left >  3
    rev64    $res0.16b, $res1.16b                                             // GHASH final-3 block
    ld1    { $res1.16b}, [$input_ptr], #16                                    // AES final-2 block - load ciphertext
    stp    $output_l0, $output_h0, [$output_ptr], #16                         // AES final-3 block  - store result
    mov    $acc_md, $h34k.d[1]                                                // GHASH final-3 block - mid
    eor    $res0.16b, $res0.16b, $t0.16b                                      // feed in partial tag
    eor    $ctr0.16b, $res1.16b, $ctr1.16b                                    // AES final-2 block - result
    mov    $rk4d, $res0.d[1]                                                  // GHASH final-3 block - mid
    mov    $output_l0, $ctr0.d[0]                                             // AES final-2 block - mov low
    mov    $output_h0, $ctr0.d[1]                                             // AES final-2 block - mov high
    eor    $rk4v.8b, $rk4v.8b, $res0.8b                                       // GHASH final-3 block - mid
    movi    $t0.8b, #0                                                        // suppress further partial tag feed in
    pmull2    $acc_h.1q, $res0.2d, $h4.2d                                     // GHASH final-3 block - high
    pmull    $acc_m.1q, $rk4v.1d, $acc_m.1d                                   // GHASH final-3 block - mid
    eor    $output_l0, $output_l0, $rkN_l                                     // AES final-2 block - round N low
    pmull    $acc_l.1q, $res0.1d, $h4.1d                                      // GHASH final-3 block - low
    eor    $output_h0, $output_h0, $rkN_h                                     // AES final-2 block - round N high
.Ldec_blocks_more_than_2:                                                     // blocks left >  2
    rev64    $res0.16b, $res1.16b                                             // GHASH final-2 block
    ld1    { $res1.16b}, [$input_ptr], #16                                    // AES final-1 block - load ciphertext
    eor    $res0.16b, $res0.16b, $t0.16b                                      // feed in partial tag
    stp    $output_l0, $output_h0, [$output_ptr], #16                         // AES final-2 block  - store result
    eor    $ctr0.16b, $res1.16b, $ctr2.16b                                    // AES final-1 block - result
    mov    $rk4d, $res0.d[1]                                                  // GHASH final-2 block - mid
    pmull    $rk3q1, $res0.1d, $h3.1d                                         // GHASH final-2 block - low
    pmull2    $rk2.1q, $res0.2d, $h3.2d                                       // GHASH final-2 block - high
    eor    $rk4v.8b, $rk4v.8b, $res0.8b                                       // GHASH final-2 block - mid
    mov    $output_l0, $ctr0.d[0]                                             // AES final-1 block - mov low
    mov    $output_h0, $ctr0.d[1]                                             // AES final-1 block - mov high
    eor    $acc_l.16b, $acc_l.16b, $rk3.16b                                   // GHASH final-2 block - low
    movi    $t0.8b, #0                                                        // suppress further partial tag feed in
    pmull    $rk4v.1q, $rk4v.1d, $h34k.1d                                     // GHASH final-2 block - mid
    eor    $acc_h.16b, $acc_h.16b, $rk2.16b                                   // GHASH final-2 block - high
    eor    $output_l0, $output_l0, $rkN_l                                     // AES final-1 block - round N low
    eor    $acc_m.16b, $acc_m.16b, $rk4v.16b                                  // GHASH final-2 block - mid
    eor    $output_h0, $output_h0, $rkN_h                                     // AES final-1 block - round N high
.Ldec_blocks_more_than_1:                                                     // blocks left >  1
    stp    $output_l0, $output_h0, [$output_ptr], #16                         // AES final-1 block  - store result
    rev64    $res0.16b, $res1.16b                                             // GHASH final-1 block
    ld1    { $res1.16b}, [$input_ptr], #16                                    // AES final block - load ciphertext
    eor    $res0.16b, $res0.16b, $t0.16b                                      // feed in partial tag
    movi    $t0.8b, #0                                                        // suppress further partial tag feed in
    mov    $rk4d, $res0.d[1]                                                  // GHASH final-1 block - mid
    eor    $ctr0.16b, $res1.16b, $ctr3.16b                                    // AES final block - result
    pmull2    $rk2q1, $res0.2d, $h2.2d                                        // GHASH final-1 block - high
    eor    $rk4v.8b, $rk4v.8b, $res0.8b                                       // GHASH final-1 block - mid
    pmull    $rk3q1, $res0.1d, $h2.1d                                         // GHASH final-1 block - low
    mov    $output_l0, $ctr0.d[0]                                             // AES final block - mov low
    ins    $rk4v.d[1], $rk4v.d[0]                                             // GHASH final-1 block - mid
    mov    $output_h0, $ctr0.d[1]                                             // AES final block - mov high
    pmull2    $rk4v.1q, $rk4v.2d, $h12k.2d                                    // GHASH final-1 block - mid
    eor    $output_l0, $output_l0, $rkN_l                                     // AES final block - round N low
    eor    $acc_l.16b, $acc_l.16b, $rk3.16b                                   // GHASH final-1 block - low
    eor    $acc_h.16b, $acc_h.16b, $rk2.16b                                   // GHASH final-1 block - high
    eor    $acc_m.16b, $acc_m.16b, $rk4v.16b                                  // GHASH final-1 block - mid
    eor    $output_h0, $output_h0, $rkN_h                                     // AES final block - round N high
.Ldec_blocks_less_than_1:                                                     // blocks left <= 1
    add    $T32, $T32, $bit_length, lsr #7                                    // Calculate the updated counter based on the number of 16B chunks we processed
    rev    $W32, $W32
    str    $W32, [$counter, #12]                                              // store the updated counter
    and    $bit_length, $bit_length, #127                                     // bit_length %= 128
    mvn    $rkN_h, xzr                                                        // rkN_h = 0xffffffffffffffff
    sub    $bit_length, $bit_length, #128                                     // bit_length -= 128
    mvn    $rkN_l, xzr                                                        // rkN_l = 0xffffffffffffffff
    ldp    $end_input_ptr, $main_end_input_ptr, [$output_ptr]                 // load existing bytes we need to not overwrite
    neg    $bit_length, $bit_length                                           // bit_length = 128 - #bits in input (in range [1,128])
    and    $bit_length, $bit_length, #127                                     // bit_length %= 128
    lsr    $rkN_h, $rkN_h, $bit_length                                        // rkN_h is mask for top 64b of last block
    cmp    $bit_length, #64
    csel    $ctr32x, $rkN_l, $rkN_h, lt
    csel    $ctr96_b64x, $rkN_h, xzr, lt
    fmov    $ctr0d, $ctr32x                                                   // ctr0b is mask for last block
    and    $output_l0, $output_l0, $ctr32x
    mov    $ctr0.d[1], $ctr96_b64x
    bic    $end_input_ptr, $end_input_ptr, $ctr32x                            // mask out low existing bytes
    bic    $main_end_input_ptr, $main_end_input_ptr, $ctr96_b64x              // mask out high existing bytes
    orr    $output_l0, $output_l0, $end_input_ptr
    and    $output_h0, $output_h0, $ctr96_b64x
    orr    $output_h0, $output_h0, $main_end_input_ptr
    and    $res1.16b, $res1.16b, $ctr0.16b                                    // possibly partial last block has zeroes in highest bits
    rev64    $res0.16b, $res1.16b                                             // GHASH final block
    eor    $res0.16b, $res0.16b, $t0.16b                                      // feed in partial tag
    pmull    $rk3q1, $res0.1d, $h1.1d                                         // GHASH final block - low
    mov    $t0d, $res0.d[1]                                                   // GHASH final block - mid
    eor    $t0.8b, $t0.8b, $res0.8b                                           // GHASH final block - mid
    pmull2    $rk2q1, $res0.2d, $h1.2d                                        // GHASH final block - high
    pmull    $t0.1q, $t0.1d, $h12k.1d                                         // GHASH final block - mid
    eor    $acc_h.16b, $acc_h.16b, $rk2.16b                                   // GHASH final block - high
    eor    $acc_l.16b, $acc_l.16b, $rk3.16b                                   // GHASH final block - low
    eor    $acc_m.16b, $acc_m.16b, $t0.16b                                    // GHASH final block - mid
    ldr    $mod_constantd, [sp, #$mod_constant_sp_offset]
    eor    $t9.16b, $acc_l.16b, $acc_h.16b                                    // MODULO - karatsuba tidy up
    eor    $acc_m.16b, $acc_m.16b, $t9.16b                                    // MODULO - karatsuba tidy up
    pmull    $mod_t.1q, $acc_h.1d, $mod_constant.1d                           // MODULO - top 64b align with mid
    ext    $acc_h.16b, $acc_h.16b, $acc_h.16b, #8                             // MODULO - other top alignment
    eor    $acc_m.16b, $acc_m.16b, $mod_t.16b                                 // MODULO - fold into mid
    eor    $acc_m.16b, $acc_m.16b, $acc_h.16b                                 // MODULO - fold into mid
    pmull    $mod_constant.1q, $acc_m.1d, $mod_constant.1d                    // MODULO - mid 64b align with low
    ext    $acc_m.16b, $acc_m.16b, $acc_m.16b, #8                             // MODULO - other mid alignment
    eor    $acc_l.16b, $acc_l.16b, $mod_constant.16b                          // MODULO - fold into low
    stp    $output_l0, $output_h0, [$output_ptr]
    eor    $acc_l.16b, $acc_l.16b, $acc_m.16b                                 // MODULO - fold into low
    ext    $acc_l.16b, $acc_l.16b, $acc_l.16b, #8
    rev64    $acc_l.16b, $acc_l.16b                                           // Final Tag
    mov    x0, $len
    st1    { $acc_l.16b }, [$current_tag]                                     // Store final tag
    ldp    x19, x20, [sp, #16]
    ldp    x21, x22, [sp, #32]
    ldp    x23, x24, [sp, #48]
    ldp    d8, d9, [sp, #64]
    ldp    d10, d11, [sp, #80]
    ldp    d12, d13, [sp, #96]
    ldp    d14, d15, [sp, #112]
    ldp    x29, x30, [sp], #224
    AARCH64_VALIDATE_LINK_REGISTER
    ret
.size aes_gcm_dec_kernel,.-aes_gcm_dec_kernel
___
    # 1. Print directives
    print $header_directives;

    # 2. Print Legacy implementation
    print get_transformed_code(0, $code_template);

    print "\n";

    # 3. Print EOR3 implementation
    print get_transformed_code(1, $code_template);

    print <<'___';
#endif  // __ARM_MAX_ARCH__ >= 8
___

    # Close the handle (which flushes to arm-xlate.pl)
    close STDOUT or die "error closing STDOUT: $!";
}
}