AES-GCM: optimize ARMv8 kernel and add EOR3 support Originally written by Jamison Collins <jdcollin@google.com> in cl/828730455. The assembly parts of this CL were reviewed there. go/aes-gcm-eor3-benchmarks has some (internal) benchmarks. I've also added some on the hardware I had on hand here. Jamison's notes: This change optimizes the existing ARMv8 AES-GCM assembly by reducing NEON register pressure and introduces a new kernel variant leveraging the EOR3 instruction. This CL improves performance by up to 12% on Conan, 28% on Athena, and 41% on Bondi Beach. Core Assembly Optimizations: Significant instruction rescheduling was implemented in aesv8-gcm-armv8.pl to reduce false dependencies and better utilize execution ports. To address NEON bottlenecks, operations were shifted away from NEON registers where possible. Counter Management: A key optimization is the reworked counter management. Previously, counters were incremented, reversed, and moved from GPR to NEON registers within the hot loop for every block. The new approach precalculates these values for the subsequent iteration and stores them on the stack. Inside the loop, they are pulled in via NEON loads. As values are passed via memory it's necessary to calculate the counters one iteration ahead to avoid expensive LD-ST forwarding. EOR3 Support: Support for the EOR3 instruction (part of the SHA3 extension) has been piped through the library to further optimize AES-GCM on capable hardware. Detection: SHA3 capability detection was added to Linux (via hwcap), Apple (via sysctl), Fuchsia, and system register reading for baremetal/FreeBSD. Dispatch: Added gcm_arm64_aes_eor3 to the gcm_impl_t enum. CRYPTO_gcm128_init_aes_key now selects this implementation when gcm_sha3_capable() is true, enabling the optimized _eor3 encrypt/decrypt functions. Kernel: The Perl script now generates a second kernel variant utilizing EOR3 to merge XOR operations during GHASH accumulation and final text rendering. Both versions are stored under different names in aesv8-gcm-armv8-linux.S. Changes I made when extracting this to the upstream repo: - Rebased to main and reran pregenerate to pick up new symbols to prefix - Pulled the arch_extension machinery into its own CL - Went ahead and added Windows feature dispatch, since MS has added those constants now - Switched armv8_feature_parsing.h to C++ inline functions, to avoid potential theoretical ODR issues with static symbols in headers. Since these are now plain C++, I made them C++-named. - Added armv8_feature_parsing.h to the build. Benchmarks: Apple M1 Pro (has EOR3): Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------------------------------- BM_SpeedAEAD/seal_aes_128_gcm/InputSize:16 +0.0519 +0.0519 42 44 42 44 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:256 -0.0356 -0.0297 76 74 76 73 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:1350 -0.1067 -0.1064 255 228 255 228 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:8192 -0.1451 -0.1458 1253 1071 1249 1067 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:16384 -0.1427 -0.1446 2458 2108 2453 2099 BM_SpeedAEAD/open_aes_128_gcm/InputSize:16 +0.0697 +0.0725 45 48 45 48 BM_SpeedAEAD/open_aes_128_gcm/InputSize:256 -0.0254 -0.0301 79 77 79 77 BM_SpeedAEAD/open_aes_128_gcm/InputSize:1350 -0.1406 -0.1458 266 229 265 226 BM_SpeedAEAD/open_aes_128_gcm/InputSize:8192 -0.2051 -0.2061 1296 1030 1295 1028 BM_SpeedAEAD/open_aes_128_gcm/InputSize:16384 -0.2122 -0.2114 2547 2007 2541 2004 Pixel 5A (does not have EOR3, just the base AES and PMULL extensions): Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------------------------------- BM_SpeedAEAD/seal_aes_128_gcm/InputSize:16 +0.0128 +0.0125 116 118 116 118 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:256 -0.0420 -0.0422 213 204 213 204 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:1350 -0.0767 -0.0781 740 683 739 681 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:8192 -0.1032 -0.1032 3829 3434 3822 3427 BM_SpeedAEAD/seal_aes_128_gcm/InputSize:16384 -0.1077 -0.1081 7568 6753 7553 6737 BM_SpeedAEAD/open_aes_128_gcm/InputSize:16 +0.0000 -0.0002 122 122 122 122 BM_SpeedAEAD/open_aes_128_gcm/InputSize:256 +0.0125 +0.0122 211 214 211 213 BM_SpeedAEAD/open_aes_128_gcm/InputSize:1350 -0.0161 -0.0165 697 685 695 684 BM_SpeedAEAD/open_aes_128_gcm/InputSize:8192 -0.0183 -0.0183 3546 3482 3539 3475 BM_SpeedAEAD/open_aes_128_gcm/InputSize:16384 -0.0184 -0.0176 6985 6857 6968 6845 Change-Id: Ied74d3493f174f6e8aeaa300816b39d72f2be042 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/87988 Commit-Queue: David Benjamin <davidben@google.com> Reviewed-by: Lily Chen <chlily@google.com>
diff --git a/build.json b/build.json index 92cbfff..45fb7c4 100644 --- a/build.json +++ b/build.json
@@ -495,6 +495,7 @@ "include/openssl/xwing.h" ], "internal_hdrs": [ + "crypto/armv8_feature_parsing.h", "crypto/asn1/internal.h", "crypto/bio/internal.h", "crypto/bytestring/internal.h",
diff --git a/crypto/armv8_feature_parsing.h b/crypto/armv8_feature_parsing.h new file mode 100644 index 0000000..5ebb876 --- /dev/null +++ b/crypto/armv8_feature_parsing.h
@@ -0,0 +1,67 @@ +#ifndef OPENSSL_CRYPTO_CPU_ARMV8_FEATURE_PARSING_H +#define OPENSSL_CRYPTO_CPU_ARMV8_FEATURE_PARSING_H + +#include <openssl/cpu.h> +#include <stdint.h> + +#include "internal.h" + +#if defined(OPENSSL_AARCH64) + +BSSL_NAMESPACE_BEGIN +namespace armcap { + +// Common field indices based on ARM architecture specification for +// ID_AA64ISAR0_EL1. These are indices (multiplied by 4 for the bit shift). +// Note: SHA3_FIELD_IDX 8 * 4 = 32 (ID_AA64ISAR0_EL1_SHA3_SHIFT value) +#define ID_AA64ISAR0_AES_FIELD_IDX 1 // Bits [7:4] +#define ID_AA64ISAR0_SHA1_FIELD_IDX 2 // Bits [11:8] +#define ID_AA64ISAR0_SHA2_FIELD_IDX 3 // Bits [15:12] +#define ID_AA64ISAR0_SHA3_FIELD_IDX 8 // Bits [35:32] +#define NBITS_ID_FIELD 4 + +// Helper function to extract a 4-bit field based on its index. +inline unsigned GetIDField(uint64_t reg, unsigned field_idx) { + // We mask with 0xf to ensure only the 4 relevant bits are returned. + return (reg >> (field_idx * NBITS_ID_FIELD)) & 0xf; +} + +// The core function that converts the raw ID_AA64ISAR0 register value +// into the OR'd capability flags (ARMV8_AES, ARMV8_SHA3, etc.). +inline uint32_t ParseISAR0Flags(uint64_t isar0) { + uint32_t armcap = 0; + // AES and PMULL check + unsigned aes = GetIDField(isar0, ID_AA64ISAR0_AES_FIELD_IDX); + if (aes > 0) { + armcap |= ARMV8_AES; + } + if (aes > 1) { + armcap |= ARMV8_PMULL; + } + // SHA1 check + unsigned sha1 = GetIDField(isar0, ID_AA64ISAR0_SHA1_FIELD_IDX); + if (sha1 > 0) { + armcap |= ARMV8_SHA1; + } + // SHA256 and SHA512 check + unsigned sha2 = GetIDField(isar0, ID_AA64ISAR0_SHA2_FIELD_IDX); + if (sha2 > 0) { + armcap |= ARMV8_SHA256; + } + if (sha2 > 1) { + armcap |= ARMV8_SHA512; + } + // SHA3 (EOR3) check + unsigned sha3 = GetIDField(isar0, ID_AA64ISAR0_SHA3_FIELD_IDX); + if (sha3 > 0) { + armcap |= ARMV8_SHA3; + } + return armcap; +} + +} // namespace armcap +BSSL_NAMESPACE_END + +#endif // OPENSSL_AARCH64 + +#endif // OPENSSL_CRYPTO_CPU_ARMV8_FEATURE_PARSING_H
diff --git a/crypto/cpu_aarch64_apple.cc b/crypto/cpu_aarch64_apple.cc index c42ced4..c25b5ac 100644 --- a/crypto/cpu_aarch64_apple.cc +++ b/crypto/cpu_aarch64_apple.cc
@@ -67,6 +67,10 @@ has_hw_feature("hw.optional.armv8_2_sha512")) { OPENSSL_armcap_P |= ARMV8_SHA512; } + if (has_hw_feature("hw.optional.arm.FEAT_SHA3") || + has_hw_feature("hw.optional.armv8_2_sha3")) { + OPENSSL_armcap_P |= ARMV8_SHA3; + } } #endif // OPENSSL_AARCH64 && OPENSSL_APPLE && !OPENSSL_STATIC_ARMCAP
diff --git a/crypto/cpu_aarch64_fuchsia.cc b/crypto/cpu_aarch64_fuchsia.cc index 865dfea..88db3b2 100644 --- a/crypto/cpu_aarch64_fuchsia.cc +++ b/crypto/cpu_aarch64_fuchsia.cc
@@ -50,6 +50,9 @@ if (hwcap & ZX_ARM64_FEATURE_ISA_SHA512) { OPENSSL_armcap_P |= ARMV8_SHA512; } + if (hwcap & ZX_ARM64_FEATURE_ISA_SHA3) { + OPENSSL_armcap_P |= ARMV8_SHA3; + } } #endif // OPENSSL_AARCH64 && OPENSSL_FUCHSIA && !OPENSSL_STATIC_ARMCAP
diff --git a/crypto/cpu_aarch64_linux.cc b/crypto/cpu_aarch64_linux.cc index f203f11..527c7f0 100644 --- a/crypto/cpu_aarch64_linux.cc +++ b/crypto/cpu_aarch64_linux.cc
@@ -32,6 +32,7 @@ static const unsigned long kPMULL = 1 << 4; static const unsigned long kSHA1 = 1 << 5; static const unsigned long kSHA256 = 1 << 6; + static const unsigned long kSHA3 = 1 << 17; static const unsigned long kSHA512 = 1 << 21; if ((hwcap & kNEON) == 0) { @@ -57,6 +58,9 @@ if (hwcap & kSHA512) { OPENSSL_armcap_P |= ARMV8_SHA512; } + if (hwcap & kSHA3) { + OPENSSL_armcap_P |= ARMV8_SHA3; + } } #endif // OPENSSL_AARCH64 && OPENSSL_LINUX && !OPENSSL_STATIC_ARMCAP
diff --git a/crypto/cpu_aarch64_openbsd.cc b/crypto/cpu_aarch64_openbsd.cc index 64b1633..235c14a 100644 --- a/crypto/cpu_aarch64_openbsd.cc +++ b/crypto/cpu_aarch64_openbsd.cc
@@ -21,6 +21,7 @@ #include <machine/cpu.h> #include <sys/sysctl.h> +#include "armv8_feature_parsing.h" #include "internal.h" @@ -37,25 +38,8 @@ OPENSSL_armcap_P |= ARMV7_NEON; - if (ID_AA64ISAR0_AES(cpu_id) >= ID_AA64ISAR0_AES_BASE) { - OPENSSL_armcap_P |= ARMV8_AES; - } - - if (ID_AA64ISAR0_AES(cpu_id) >= ID_AA64ISAR0_AES_PMULL) { - OPENSSL_armcap_P |= ARMV8_PMULL; - } - - if (ID_AA64ISAR0_SHA1(cpu_id) >= ID_AA64ISAR0_SHA1_BASE) { - OPENSSL_armcap_P |= ARMV8_SHA1; - } - - if (ID_AA64ISAR0_SHA2(cpu_id) >= ID_AA64ISAR0_SHA2_BASE) { - OPENSSL_armcap_P |= ARMV8_SHA256; - } - - if (ID_AA64ISAR0_SHA2(cpu_id) >= ID_AA64ISAR0_SHA2_512) { - OPENSSL_armcap_P |= ARMV8_SHA512; - } + // Use the common parsing function to check other features. + OPENSSL_armcap_P |= armcap::ParseISAR0Flags(cpu_id); } #endif // OPENSSL_AARCH64 && OPENSSL_OPENBSD && !OPENSSL_STATIC_ARMCAP
diff --git a/crypto/cpu_aarch64_sysreg.cc b/crypto/cpu_aarch64_sysreg.cc index 9734cfc..0360a3d 100644 --- a/crypto/cpu_aarch64_sysreg.cc +++ b/crypto/cpu_aarch64_sysreg.cc
@@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <openssl/cpu.h> + #include "internal.h" // While Arm system registers are normally not available to userspace, FreeBSD @@ -23,12 +25,6 @@ #define ID_AA64PFR0_EL1_ADVSIMD 5 -#define ID_AA64ISAR0_EL1_AES 1 -#define ID_AA64ISAR0_EL1_SHA1 2 -#define ID_AA64ISAR0_EL1_SHA2 3 - -#define NBITS_ID_FIELD 4 - #define READ_SYSREG(name) \ ({ \ uint64_t _r; \ @@ -38,10 +34,8 @@ using namespace bssl; -static unsigned get_id_field(uint64_t reg, unsigned field) { - return (reg >> (field * NBITS_ID_FIELD)) & ((1 << NBITS_ID_FIELD) - 1); -} - +// We use the common get_id_field helper now, but need a signed variant +// for the NEON check using ID_AA64PFR0_EL1. static int get_signed_id_field(uint64_t reg, unsigned field) { unsigned value = get_id_field(reg, field); if (value & (1 << (NBITS_ID_FIELD - 1))) { @@ -64,26 +58,8 @@ uint64_t id_aa64isar0_el1 = READ_SYSREG("id_aa64isar0_el1"); - unsigned aes = get_id_field(id_aa64isar0_el1, ID_AA64ISAR0_EL1_AES); - if (aes > 0) { - armcap |= ARMV8_AES; - } - if (aes > 1) { - armcap |= ARMV8_PMULL; - } - - unsigned sha1 = get_id_field(id_aa64isar0_el1, ID_AA64ISAR0_EL1_SHA1); - if (sha1 > 0) { - armcap |= ARMV8_SHA1; - } - - unsigned sha2 = get_id_field(id_aa64isar0_el1, ID_AA64ISAR0_EL1_SHA2); - if (sha2 > 0) { - armcap |= ARMV8_SHA256; - } - if (sha2 > 1) { - armcap |= ARMV8_SHA512; - } + // Use the common parsing function to check all cryptographic features. + armcap |= armcap::ParseISAR0Flags(id_aa64isar0_el1); return armcap; }
diff --git a/crypto/cpu_aarch64_win.cc b/crypto/cpu_aarch64_win.cc index 6cdb00f..4d51bbd 100644 --- a/crypto/cpu_aarch64_win.cc +++ b/crypto/cpu_aarch64_win.cc
@@ -21,6 +21,9 @@ #include <windows.h> +#if !defined(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE 64 +#endif #if !defined(PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE) #define PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE 65 #endif @@ -41,6 +44,9 @@ if (IsProcessorFeaturePresent(PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE)) { OPENSSL_armcap_P |= ARMV8_SHA512; } + if (IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE)) { + OPENSSL_armcap_P |= ARMV8_SHA3; + } } #endif // OPENSSL_AARCH64 && OPENSSL_WINDOWS && !OPENSSL_STATIC_ARMCAP
diff --git a/crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl b/crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl index d2b587b..23084e6 100644 --- a/crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl +++ b/crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl
@@ -171,22 +171,73 @@ # EOR acc_l, acc_l, acc_h # EOR acc_l, acc_l, acc_m # -# This code was then modified to merge the AES-128-GCM, AES-192-GCM, and -# AES-256-GCM implementations into a single function to reduce size. We move the -# last two round keys into consistent registers across all sizes, as they're -# treated special. Then, after rounds 0 through 8, we added some branches to -# conditionally run rounds 9-10 (AES-192 + AES-256) and 11-12 (AES-256), before -# merging back into code which finishes up the last two rounds. +# Key Optimizations: # -# There is a mostly decision to be made around how much parallel work goes -# before or after the conditional part. We attempted to preserve the original -# scheduling where possible, but it's possible other schedulings are more -# optimal with the current ordering. +# NOTE: This implementation is heavily NEON-bound due to the large amount of +# vector computation. The original code from which this was derived often +# avoided unnecessary Load/Store operations, but at the expense of +# increased NEON pipeline pressure and more GPR <-> NEON data movements. +# The primary goal of the optimizations applied here is to reduce NEON +# pipeline pressure. This includes minimizing NEON register-to-register moves, +# GPR to NEON transfers (and vice versa), and other operations that consume +# execution slots on the NEON pipes. +# +# 1. Merged Kernels: AES-128, AES-192, and AES-256 GCM encryption and +# decryption are handled by single kernels each, reducing code size. +# Conditional branching is used for the key-size specific rounds. +# +# 2. Aggressive Pipelining: GHASH and AES operations for different blocks +# (4k to 4k+3) are heavily interleaved within the main loop. GHASH for +# block `n` starts while AES for block `n` is still in progress, and +# AES for block `n+4` begins. This hides instruction latencies. +# +# 3. Optimized Counter Handling: Instead of incrementing, reversing, and +# moving the counter to NEON registers for each of the four blocks in +# every loop iteration, we precompute and cache counter values on the +# stack. +# - The lower 96 bits of the counter are constant across the four blocks +# within an iteration. +# - The upper 32 bits (the byte-swapped incrementing part) are calculated +# for all four blocks (N, N+1, N+2, N+3) and stored on the stack at +# the start of the loop. +# - Values are calculated one loop iteration ahead of time. That is, +# immediately after loading values N-N+3 we calculate and store values +# N+4-N+7. This ensures that the data is available when loaded without +# store-load forwarding. +# - This strategy eliminates repeated scalar-to-vector transfers (FMOV) +# and scalar increment/reverse operations inside the tight loop. +# +# 4. Reduced Scalar-NEON Transfers: +# - For encryption, the final XOR with the last round key (rkN) and the +# AES-encrypted counter is done entirely in NEON registers using +# EOR or PLATFORM_EOR3, avoiding costly FMOV instructions to move +# plaintext data from GPRs to NEON after partial operations. A similar +# optimization is used for decryption. +# +# 5. EOR3 Optimization: The PLATFORM_EOR3 macro allows using the single +# EOR3 instruction (if available) to replace two EOR instructions, +# i.e., D = A ^ B ^ C. This is used in both GHASH accumulator updates +# and the final AES output XOR. Both versions of encrypt and decrypt are +# stored in the same assembly file. +# +# 6. Instruction scheduling was significantly modified both to directly improve +# performance as well as to remove false dependencies when that allowed +# additional improvements. There is still a bit of additional speedup +# possible but would be easiest to extract with platform-specific +# implementations. +# +use strict; +use warnings; -$flavour = shift; -$output = shift; +my $flavour = shift; +my $output = shift; -$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +if (!defined $flavour) { + die "Usage: $0 <flavour> [output_filename]\n"; +} + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; +my $xlate; ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or die "can't locate arm-xlate.pl"; @@ -194,1348 +245,1303 @@ open OUT, "|-", $^X, $xlate, $flavour, $output; *STDOUT=*OUT; -$code=<<___; -#if __ARM_MAX_ARCH__ >= 8 +# Converts an EOR3 pseudo-instruction into the corresponding into either the +# raw EOR3 encoding or into two EOR2 instructions. +sub process_eor3_match { + my ($vd_str, $vn_str, $vm_str, $va_str, $is_eor3) = @_; -.arch armv8-a+crypto + if ($is_eor3) { + # Directly emit the EOR3 mnemonic. + # Requires: .arch armv8.2-a+sha3 OR -march=armv8.2-a+sha3 + return sprintf(" eor3 %s.16b, %s.16b, %s.16b, %s.16b", + $vd_str, $vn_str, $vm_str, $va_str); + } else { + # Fallback: Two standard NEON EOR instructions + # Result = (Vn ^ Vm) ^ Va + # 1. Vd = Vn ^ Vm + # 2. Vd = Vd ^ Va + return sprintf(" eor %s.16b, %s.16b, %s.16b\n" . + " eor %s.16b, %s.16b, %s.16b", + $vd_str, $vn_str, $vm_str, # Step 1 + $vd_str, $vd_str, $va_str); # Step 2 + } +} + +# Generates the code string for a specific version (Standard or EOR3). +# Performs regex substitutions for EOR3 instructions and symbol renaming. +sub get_transformed_code { + my ($is_eor3, $template) = @_; + my $temp_code = $template; + + if ($is_eor3) { + # Suffix function names + $temp_code =~ s/\b(aes_gcm_enc_kernel)\b/${1}_eor3/g; + $temp_code =~ s/\b(aes_gcm_dec_kernel)\b/${1}_eor3/g; + # Suffix local labels + $temp_code =~ s/(\.L[a-zA-Z0-9_]+)/$1_eor3/g; + } + + $temp_code =~ s/PLATFORM_EOR3\s*\(\s*([vV0-9]+)\s*,\s*([vV0-9]+)\s*,\s*([vV0-9]+)\s*,\s*([vV0-9]+)\s*\)/ + process_eor3_match($1, $2, $3, $4, $is_eor3)/ge; + + return $temp_code; +} + +{ +{ + # We attempt to minimize the number of aliases for the same register but + # unfortunately sometimes you have to specify a Q or D register. + + ## AArch64 General-Purpose Registers (GPRs) + my $input_ptr = "x0"; # IN: Pointer to input plaintext + my $bit_length = "x1"; # IN: Length of input in bits + my $output_ptr = "x2"; # IN: Pointer to output ciphertext + my $current_tag = "x3"; # IN: Pointer to 16-byte GCM tag (T) + my $end_input_ptr = "x4"; # Calculated end of all input + my $main_end_input_ptr = "x5"; # Calculated end of the main 4-block loop + my $Htable = "x6"; # Pointer to GHASH key table (H) + my $key_ptr = "x7"; # Pointer to end of AES key schedule + my $cc = "x8"; # Pointer to AES key schedule (context) + + # Re-purposed registers for tail/scalar handling + my $input_l0 = "x6"; + my $input_h0 = "x7"; + + my $T32 = "x10"; # Holds scalar 64-bit counter value + my $W32 = "w10"; # Holds scalar 32-bit counter value + my $W_T32 = "w12"; # Holds reversed scalar 32-bit counter + my $rkN_l = "x13"; # Holds low 64 bits of final round key + my $rkN_h = "x14"; # Holds high 64 bits of final round key + my $len = "x15"; # Byte length of input + my $counter = "x16"; # Holds pointer to the 16-byte counter block (CTR) + my $rounds = "x17"; # Number of AES rounds (scalar) + my $rounds_w = "w17"; # Number of AES rounds (scalar) + my $tmp_gpr_w = "w20"; # Scratch register (word) + my $mod_constantx = "x21"; # Holds GHASH modulus 0xc2... (scalar) + + ## NEON/SIMD Registers (v0-v31) + my ($ctr0, $ctr1, $ctr2, $ctr3) = map("v$_", (0..3)); + my ($ctr0q, $ctr1q, $ctr2q, $ctr3q) = map("q$_", (0..3)); + my $ctr0d = "d0"; + + my ($res0, $res1, $res2, $res3) = map("v$_", (4..7)); + my ($res0q, $res1q, $res2q, $res3q) = map("q$_", (4..7)); + my ($res0d, $res1d, $res2d, $res3d) = map("d$_", (4..7)); + + my ($acc_h, $acc_m, $acc_l) = map("v$_", (9..11)); + + my ($h1, $h2, $h3, $h4) = map("v$_", (12..15)); + my ($h1q, $h2q, $h3q, $h4q) = map("q$_", (12..15)); + + my ($h12k, $h34k) = map("v$_", (16..17)); # H[1]^H[2] and H[3]^H[4] + + my ($rk0, $rk1, $rk2, $rk3, $rk4, $rk5, $rk6, $rk7, $rk8) = map("v$_", (18..26)); + my ($rk0q, $rk1q, $rk2q, $rk3q, $rk4q, $rk5q, $rk6q, $rk7q, $rk8q) = map("q$_", (18..26)); + + my $rk9_11_tmp = "v27"; # Used for rk9 (if AES-192) and rk11 (if AES-256) + my $rk10_12 = "v28"; # Used for rk10 (if AES-192) and rk12 (if AES-256) + my $rk9_11_tmpq = "q27"; + my $rk10_12q = "q28"; + + my $rkNm1 = "v31"; # Round N-1 key + my $rkNm1q = "q31"; + + my $mod_constant = "v8"; + my $mod_constantd = "d8"; + + my $final_block_dest = "v18"; + my $final_block_destq = "q18"; + + my ($ghash_t0, $ghash_t1, $ghash_t2, $ghash_t3) = map("v$_", (20..23)); + my ($ghash_t0d, $ghash_t1d, $ghash_t2d, $ghash_t3d) = map("d$_", (20..23)); + + my $v_rkN = "v30"; + my $v_rkNq = "q30"; + + # Locations on the stack + my $mod_constant_sp_offset = 128; + my $ctr0_sp_offset = 160; + my $ctr1_sp_offset = 176; + my $ctr2_sp_offset = 192; + my $ctr3_sp_offset = 208; + + # Registers specific to dec_kernel + my ($input_l1, $input_h1) = map("x$_", (19..20)); + my $ctr32w = "w9"; + my ($ctr32x, $ctr96_b64x) = map("x$_", (9..10)); + my $acc_md = "d10"; + + # Decryption temporary NEON registers + my $t0 = "v8"; my $t0d = "d8"; + my $t1 = "v4"; my $t1d = "d4"; + my $t2 = "v8"; my $t2d = "d8"; + my $t3 = "v4"; my $t3d = "d4"; + my $t4 = "v4"; my $t4d = "d4"; + my $t5 = "v5"; my $t5d = "d5"; + my $t6 = "v8"; my $t6d = "d8"; + my $t7 = "v5"; my $t7d = "d5"; + my $t8 = "v4"; my $t8d = "d8"; + my $t9 = "v6"; my $t9d = "d6"; + + my ($ctr_t0, $ctr_t1, $ctr_t2, $ctr_t3) = map("v$_", (4..7)); + my $mod_t = "v7"; + + # Decryption key registers + my $rk2q1 = "v20.1q"; + my $rk3q1 = "v21.1q"; + my $rk4v = "v22"; + my $rk4d = "d22"; + + my ($output_l1, $output_h1, $output_l2, $output_h2, $output_l3, $output_h3) = map("x$_", (19..24)); + my ($output_l0, $output_h0) = map("x$_", (6..7)); + + # --- Assemble the entire template --- + my $code_template = ""; + my $header_directives = <<'___'; +#if __ARM_MAX_ARCH__ >= 8 +.arch armv8.2-a+crypto+sha3 .text ___ -$input_ptr="x0"; #argument block -$bit_length="x1"; -$output_ptr="x2"; -$current_tag="x3"; -$Htable="x6"; -$counter="x16"; -$cc="x8"; - -{ -my ($end_input_ptr,$main_end_input_ptr,$input_l0,$input_h0)=map("x$_",(4..7)); -my ($input_l1,$input_h1,$input_l2,$input_h2,$input_l3,$input_h3)=map("x$_",(19..24)); -my ($output_l1,$output_h1,$output_l2,$output_h2,$output_l3,$output_h3)=map("x$_",(19..24)); -my ($output_l0,$output_h0)=map("x$_",(6..7)); - -# rkN_l and rkN_h store the final round key, which is handled slightly -# differently because it is EORed through general-purpose registers. -my $ctr32w="w9"; -my ($ctr32x,$ctr96_b64x,$ctr96_t32x,$rctr32x,$rkN_l,$rkN_h,$len)=map("x$_",(9..15)); -my ($ctr96_t32w,$rctr32w)=map("w$_",(11..12)); - -my $rounds="x17"; -my $roundsw="w17"; - -my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$res0b,$res1b,$res2b,$res3b)=map("v$_.16b",(0..7)); -my ($ctr0,$ctr1,$ctr2,$ctr3,$res0,$res1,$res2,$res3)=map("v$_",(0..7)); -my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$res0d,$res1d,$res2d,$res3d)=map("d$_",(0..7)); -my ($res0q,$res1q,$res2q,$res3q)=map("q$_",(4..7)); - -my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(9..11)); -my ($acc_h,$acc_m,$acc_l)=map("v$_",(9..11)); -my ($acc_hd,$acc_md,$acc_ld)=map("d$_",(9..11)); - -my ($h1,$h2,$h3,$h4,$h12k,$h34k)=map("v$_",(12..17)); -my ($h1q,$h2q,$h3q,$h4q)=map("q$_",(12..15)); -my ($h1b,$h2b,$h3b,$h4b)=map("v$_.16b",(12..15)); - -my $t0="v8"; -my $t0d="d8"; -my $t1="v4"; -my $t1d="d4"; -my $t2="v8"; -my $t2d="d8"; -my $t3="v4"; -my $t3d="d4"; -my $t4="v4"; -my $t4d="d4"; -my $t5="v5"; -my $t5d="d5"; -my $t6="v8"; -my $t6d="d8"; -my $t7="v5"; -my $t7d="d5"; -my $t8="v6"; -my $t8d="d6"; -my $t9="v4"; -my $t9d="d4"; - -my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3)=map("v$_",(4..7)); -my ($ctr_t0d,$ctr_t1d,$ctr_t2d,$ctr_t3d)=map("d$_",(4..7)); -my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b)=map("v$_.16b",(4..7)); - -my $mod_constantd="d8"; -my $mod_constant="v8"; -my $mod_t="v7"; - -# rkNm1 stores the second-to-last round key, which is handled slightly -# differently because it uses plain AESE instead of an AESE + AESMC macro-op. -my ($rk0,$rk1,$rk2,$rk3,$rk4,$rk5,$rk6,$rk7,$rk8,$rk9,$rk10,$rk11,$rk12,$rkNm1)=map("v$_.16b",(18..31)); -my ($rk0q,$rk1q,$rk2q,$rk3q,$rk4q,$rk5q,$rk6q,$rk7q,$rk8q,$rk9q,$rk10q,$rk11q,$rk12q,$rkNm1q)=map("q$_",(18..31)); -my $rk2q1="v20.1q"; -my $rk3q1="v21.1q"; -my $rk4v="v22"; -my $rk4d="d22"; - -################################################################################ -# size_t aes_gcm_enc_kernel(const uint8_t *in, -# size_t len_bits, -# uint8_t *out, -# u64 *Xi, -# uint8_t ivec[16], -# const void *key, -# const void *Htable); -# -$code.=<<___; + $code_template .= <<"___"; .global aes_gcm_enc_kernel .type aes_gcm_enc_kernel,%function .align 4 aes_gcm_enc_kernel: - AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! - mov x29, sp - stp x19, x20, [sp, #16] - mov $counter, x4 - mov $cc, x5 - stp x21, x22, [sp, #32] - stp x23, x24, [sp, #48] - stp d8, d9, [sp, #64] - stp d10, d11, [sp, #80] - stp d12, d13, [sp, #96] - stp d14, d15, [sp, #112] - ldr $roundsw, [$cc, #240] - add $input_l1, $cc, $rounds, lsl #4 // borrow input_l1 for last key - ldp $rkN_l, $rkN_h, [$input_l1] // load round N keys - ldr $rkNm1q, [$input_l1, #-16] // load round N-1 keys - add $end_input_ptr, $input_ptr, $bit_length, lsr #3 // end_input_ptr - lsr $main_end_input_ptr, $bit_length, #3 // byte_len - mov $len, $main_end_input_ptr - ldp $ctr96_b64x, $ctr96_t32x, [$counter] // ctr96_b64, ctr96_t32 - ld1 { $ctr0b}, [$counter] // special case vector load initial counter so we can start first AES block as quickly as possible - sub $main_end_input_ptr, $main_end_input_ptr, #1 // byte_len - 1 - ldr $rk0q, [$cc, #0] // load rk0 - and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - ldr $rk7q, [$cc, #112] // load rk7 - add $main_end_input_ptr, $main_end_input_ptr, $input_ptr - lsr $rctr32x, $ctr96_t32x, #32 - fmov $ctr2d, $ctr96_b64x // CTR block 2 - orr $ctr96_t32w, $ctr96_t32w, $ctr96_t32w - rev $rctr32w, $rctr32w // rev_ctr32 - fmov $ctr1d, $ctr96_b64x // CTR block 1 - aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 0 - add $rctr32w, $rctr32w, #1 // increment rev_ctr32 - rev $ctr32w, $rctr32w // CTR block 1 - fmov $ctr3d, $ctr96_b64x // CTR block 3 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 1 - add $rctr32w, $rctr32w, #1 // CTR block 1 - ldr $rk1q, [$cc, #16] // load rk1 - fmov $ctr1.d[1], $ctr32x // CTR block 1 - rev $ctr32w, $rctr32w // CTR block 2 - add $rctr32w, $rctr32w, #1 // CTR block 2 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 2 - ldr $rk2q, [$cc, #32] // load rk2 - fmov $ctr2.d[1], $ctr32x // CTR block 2 - rev $ctr32w, $rctr32w // CTR block 3 - aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 1 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 3 - fmov $ctr3.d[1], $ctr32x // CTR block 3 - aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 0 - ldr $rk3q, [$cc, #48] // load rk3 - aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 2 - ldr $rk6q, [$cc, #96] // load rk6 - aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 0 - ldr $rk5q, [$cc, #80] // load rk5 - aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 1 - ldr $h3q, [$Htable, #48] // load h3l | h3h - ext $h3b, $h3b, $h3b, #8 - aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 0 - aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 1 - ldr $rk4q, [$cc, #64] // load rk4 - aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 2 - ldr $h2q, [$Htable, #32] // load h2l | h2h - ext $h2b, $h2b, $h2b, #8 - aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 1 - ldr $rk12q, [$cc, #192] // load rk12 - aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 2 - ldr $h4q, [$Htable, #80] // load h4l | h4h - ext $h4b, $h4b, $h4b, #8 - aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 3 - ldr $rk11q, [$cc, #176] // load rk11 - aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 2 - ldr $rk8q, [$cc, #128] // load rk8 - aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 3 - add $rctr32w, $rctr32w, #1 // CTR block 3 - aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 3 - aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 3 - ld1 { $acc_lb}, [$current_tag] - ext $acc_lb, $acc_lb, $acc_lb, #8 - rev64 $acc_lb, $acc_lb - aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 4 - aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 4 - aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 4 - aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 4 - cmp $rounds, #12 // setup flags for AES-128/192/256 check - aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 5 - aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 5 - aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 5 - aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 5 - aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 6 - trn2 $h34k.2d, $h3.2d, $h4.2d // h4l | h3l - aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 6 - ldr $rk9q, [$cc, #144] // load rk9 - aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 6 - ldr $h1q, [$Htable] // load h1l | h1h - ext $h1b, $h1b, $h1b, #8 - aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 6 - ldr $rk10q, [$cc, #160] // load rk10 - aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 7 - trn1 $acc_h.2d, $h3.2d, $h4.2d // h4h | h3h - aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 7 - aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 7 - aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 7 - trn2 $h12k.2d, $h1.2d, $h2.2d // h2l | h1l - aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 8 - aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 8 - aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 8 - aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 8 - b.lt .Lenc_finish_first_blocks // branch if AES-128 - - aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 9 - aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 9 - aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 9 - aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 9 - aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 10 - aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 10 - aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 10 - aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 10 - b.eq .Lenc_finish_first_blocks // branch if AES-192 - - aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 11 - aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 11 - aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 11 - aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 11 - aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 12 - aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 12 - aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 12 - aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 12 - + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + ld1 { $ctr0.16b}, [x4] // Load initial counter block + stp x19, x20, [sp, #16] + mov $ctr1.16b, $ctr0.16b // Initialize ctr1-3 from ctr0 + mov $ctr2.16b, $ctr0.16b + mov $ctr3.16b, $ctr0.16b + mov $counter, x4 // Pointer to counter block in memory + mov $cc, x5 // Pointer to AES key schedule context + stp $mod_constantx, x22, [sp, #32] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel + stp d8, d9, [sp, #64] // Save Neon registers + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr $rounds_w, [$cc, #240] // Load number of AES rounds + add $key_ptr, $cc, $rounds, lsl #4 // Calculate pointer to the last round key + ldp $rkN_l, $rkN_h, [$key_ptr] // load round N key (for final XOR) + ldr $rkNm1q, [$key_ptr, #-16] // load round N-1 key + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 // Calculate end of input + lsr $main_end_input_ptr, $bit_length, #3 // Total byte length + mov $len, $main_end_input_ptr + ldr $W_T32, [$counter, #12] // Load counter's low 32 bits + sub $main_end_input_ptr, $main_end_input_ptr, #1 // byte_len - 1 + ldr $rk0q, [$cc, #0] // load rk0 + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + rev $W_T32, $W_T32 // Reverse for big-endian increment + uxtw $T32, $W_T32 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov $mod_constantx, #0xc200000000000000 // GHASH reduction constant + str $mod_constantx, [sp, #$mod_constant_sp_offset] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., $ctr0_sp_offset + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str $ctr0q, [sp, #$ctr0_sp_offset] // Store base counter for block 0-3 + str $ctr0q, [sp, #$ctr1_sp_offset] + str $ctr0q, [sp, #$ctr2_sp_offset] + str $ctr0q, [sp, #$ctr3_sp_offset] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev $tmp_gpr_w, $W_T32 + mov $ctr0.s[3], $tmp_gpr_w // ctr0 + 0 + add $tmp_gpr_w, $W_T32, #1 + rev $tmp_gpr_w, $tmp_gpr_w + mov $ctr1.s[3], $tmp_gpr_w // ctr0 + 1 + add $tmp_gpr_w, $W_T32, #2 + rev $tmp_gpr_w, $tmp_gpr_w + mov $ctr2.s[3], $tmp_gpr_w // ctr0 + 2 + add $tmp_gpr_w, $W_T32, #3 + rev $tmp_gpr_w, $tmp_gpr_w + mov $ctr3.s[3], $tmp_gpr_w // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add $tmp_gpr_w, $W_T32, #4 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}] // ctr0 + 4 for next iter + add $tmp_gpr_w, $W_T32, #5 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}] // ctr0 + 5 for next iter + add $tmp_gpr_w, $W_T32, #6 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}] // ctr0 + 6 for next iter + add $tmp_gpr_w, $W_T32, #7 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}] // ctr0 + 7 for next iter + add $W_T32, $W_T32, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- + aese $ctr0.16b, $rk0.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 0 + ldp $rk1q, $rk2q, [$cc, #16] // load rk1, rk2 + aese $ctr1.16b, $rk0.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 0 + ldp $rk3q, $rk4q, [$cc, #48] // load rk3, rk4 + aese $ctr2.16b, $rk0.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 0 + ldp $rk5q, $rk6q, [$cc, #80] // load rk5, rk6 + aese $ctr3.16b, $rk0.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 0 + ldp $h2q, $h3q, [$Htable, #32] // load H2, H3 (GHASH keys) + aese $ctr0.16b, $rk1.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 1 + ldp $rk7q, $rk8q, [$cc, #112] // load rk7, rk8 + aese $ctr1.16b, $rk1.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 1 + aese $ctr2.16b, $rk1.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 1 + aese $ctr3.16b, $rk1.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 1 + aese $ctr0.16b, $rk2.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 2 + ext $h3.16b, $h3.16b, $h3.16b, #8 // Byte swap H3 for GHASH + aese $ctr1.16b, $rk2.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 2 + ext $h2.16b, $h2.16b, $h2.16b, #8 // Byte swap H2 for GHASH + aese $ctr2.16b, $rk2.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 2 + ldr $h4q, [$Htable, #80] // load H4 + aese $ctr3.16b, $rk2.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 2 + ext $h4.16b, $h4.16b, $h4.16b, #8 // Byte swap H4 for GHASH + aese $ctr0.16b, $rk3.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 3 + ld1 { $acc_l.16b}, [$current_tag] // Load initial GHASH accumulator (T) + aese $ctr1.16b, $rk3.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 3 + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 // Byte swap T for GHASH + aese $ctr2.16b, $rk3.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 3 + rev64 $acc_l.16b, $acc_l.16b // Correct byte order within 64-bit lanes + aese $ctr3.16b, $rk3.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 3 + trn2 $h34k.2d, $h3.2d, $h4.2d // Karatsuba key: H4_low | H3_low + aese $ctr0.16b, $rk4.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 4 + ldr $h1q, [$Htable] // load H1 + aese $ctr1.16b, $rk4.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 4 + ext $h1.16b, $h1.16b, $h1.16b, #8 // Byte swap H1 for GHASH + aese $ctr2.16b, $rk4.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 4 + trn1 $acc_h.2d, $h3.2d, $h4.2d // Karatsuba key: H4_high | H3_high + aese $ctr3.16b, $rk4.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 4 + trn2 $h12k.2d, $h1.2d, $h2.2d // Karatsuba key: H2_low | H1_low + aese $ctr0.16b, $rk5.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 5 + ldr $v_rkNq, [$key_ptr] // Preload round N key for final EOR + aese $ctr1.16b, $rk5.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 5 + aese $ctr3.16b, $rk5.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 5 + aese $ctr2.16b, $rk5.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 5 + aese $ctr0.16b, $rk6.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 6 + aese $ctr1.16b, $rk6.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 6 + aese $ctr2.16b, $rk6.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 6 + aese $ctr3.16b, $rk6.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 6 + aese $ctr0.16b, $rk7.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 7 + aese $ctr1.16b, $rk7.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 7 + aese $ctr2.16b, $rk7.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 7 + aese $ctr3.16b, $rk7.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 7 + aese $ctr0.16b, $rk8.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 8 + aese $ctr1.16b, $rk8.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 8 + aese $ctr2.16b, $rk8.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 8 + aese $ctr3.16b, $rk8.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 8 + cmp $rounds, #12 // setup flags for AES-128/192/256 check + b.lt .Lenc_finish_first_blocks // branch if AES-128 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #144] // load rk9, rk10 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 9 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 9 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 9 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 9 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 10 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 10 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 10 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 10 + b.eq .Lenc_finish_first_blocks // branch if AES-192 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #176] // load rk11, rk12 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 11 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 11 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 11 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 11 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 12 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 12 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 12 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 12 .Lenc_finish_first_blocks: - cmp $input_ptr, $main_end_input_ptr // check if we have <= 4 blocks - eor $h34k.16b, $h34k.16b, $acc_h.16b // h4k | h3k - aese $ctr2b, $rkNm1 // AES block 2 - round N-1 - trn1 $t0.2d, $h1.2d, $h2.2d // h2h | h1h - aese $ctr1b, $rkNm1 // AES block 1 - round N-1 - aese $ctr0b, $rkNm1 // AES block 0 - round N-1 - aese $ctr3b, $rkNm1 // AES block 3 - round N-1 - eor $h12k.16b, $h12k.16b, $t0.16b // h2k | h1k - b.ge .Lenc_tail // handle tail - - ldp $input_l1, $input_h1, [$input_ptr, #16] // AES block 1 - load plaintext - rev $ctr32w, $rctr32w // CTR block 4 - ldp $input_l0, $input_h0, [$input_ptr, #0] // AES block 0 - load plaintext - ldp $input_l3, $input_h3, [$input_ptr, #48] // AES block 3 - load plaintext - ldp $input_l2, $input_h2, [$input_ptr, #32] // AES block 2 - load plaintext - add $input_ptr, $input_ptr, #64 // AES input_ptr update - eor $input_l1, $input_l1, $rkN_l // AES block 1 - round N low - eor $input_h1, $input_h1, $rkN_h // AES block 1 - round N high - fmov $ctr_t1d, $input_l1 // AES block 1 - mov low - eor $input_l0, $input_l0, $rkN_l // AES block 0 - round N low - eor $input_h0, $input_h0, $rkN_h // AES block 0 - round N high - eor $input_h3, $input_h3, $rkN_h // AES block 3 - round N high - fmov $ctr_t0d, $input_l0 // AES block 0 - mov low - cmp $input_ptr, $main_end_input_ptr // check if we have <= 8 blocks - fmov $ctr_t0.d[1], $input_h0 // AES block 0 - mov high - eor $input_l3, $input_l3, $rkN_l // AES block 3 - round N low - eor $input_l2, $input_l2, $rkN_l // AES block 2 - round N low - fmov $ctr_t1.d[1], $input_h1 // AES block 1 - mov high - fmov $ctr_t2d, $input_l2 // AES block 2 - mov low - add $rctr32w, $rctr32w, #1 // CTR block 4 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4 - fmov $ctr_t3d, $input_l3 // AES block 3 - mov low - eor $input_h2, $input_h2, $rkN_h // AES block 2 - round N high - fmov $ctr_t2.d[1], $input_h2 // AES block 2 - mov high - eor $res0b, $ctr_t0b, $ctr0b // AES block 0 - result - fmov $ctr0d, $ctr96_b64x // CTR block 4 - fmov $ctr0.d[1], $ctr32x // CTR block 4 - rev $ctr32w, $rctr32w // CTR block 5 - add $rctr32w, $rctr32w, #1 // CTR block 5 - eor $res1b, $ctr_t1b, $ctr1b // AES block 1 - result - fmov $ctr1d, $ctr96_b64x // CTR block 5 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 5 - fmov $ctr1.d[1], $ctr32x // CTR block 5 - rev $ctr32w, $rctr32w // CTR block 6 - st1 { $res0b}, [$output_ptr], #16 // AES block 0 - store result - fmov $ctr_t3.d[1], $input_h3 // AES block 3 - mov high - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 6 - eor $res2b, $ctr_t2b, $ctr2b // AES block 2 - result - st1 { $res1b}, [$output_ptr], #16 // AES block 1 - store result - add $rctr32w, $rctr32w, #1 // CTR block 6 - fmov $ctr2d, $ctr96_b64x // CTR block 6 - fmov $ctr2.d[1], $ctr32x // CTR block 6 - st1 { $res2b}, [$output_ptr], #16 // AES block 2 - store result - rev $ctr32w, $rctr32w // CTR block 7 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 7 - eor $res3b, $ctr_t3b, $ctr3b // AES block 3 - result - st1 { $res3b}, [$output_ptr], #16 // AES block 3 - store result - b.ge .Lenc_prepretail // do prepretail - -.Lenc_main_loop: // main loop start - aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 0 - rev64 $res0b, $res0b // GHASH block 4k (only t0 is free) - aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 0 - fmov $ctr3d, $ctr96_b64x // CTR block 4k+3 - aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 0 - ext $acc_lb, $acc_lb, $acc_lb, #8 // PRE 0 - aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 1 - fmov $ctr3.d[1], $ctr32x // CTR block 4k+3 - aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 1 - ldp $input_l3, $input_h3, [$input_ptr, #48] // AES block 4k+7 - load plaintext - aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 1 - ldp $input_l2, $input_h2, [$input_ptr, #32] // AES block 4k+6 - load plaintext - aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 2 - eor $res0b, $res0b, $acc_lb // PRE 1 - aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 2 - aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 0 - eor $input_l3, $input_l3, $rkN_l // AES block 4k+7 - round N low - aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 3 - mov $acc_md, $h34k.d[1] // GHASH block 4k - mid - pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 4k - high - eor $input_h2, $input_h2, $rkN_h // AES block 4k+6 - round N high - mov $t0d, $res0.d[1] // GHASH block 4k - mid - aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 1 - rev64 $res1b, $res1b // GHASH block 4k+1 (t0 and t1 free) - aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 4 - pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 4k - low - eor $t0.8b, $t0.8b, $res0.8b // GHASH block 4k - mid - aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 2 - aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 5 - rev64 $res3b, $res3b // GHASH block 4k+3 (t0, t1, t2 and t3 free) - pmull2 $t1.1q, $res1.2d, $h3.2d // GHASH block 4k+1 - high - pmull $acc_m.1q, $t0.1d, $acc_m.1d // GHASH block 4k - mid - rev64 $res2b, $res2b // GHASH block 4k+2 (t0, t1, and t2 free) - pmull $t2.1q, $res1.1d, $h3.1d // GHASH block 4k+1 - low - eor $acc_hb, $acc_hb, $t1.16b // GHASH block 4k+1 - high - mov $t3d, $res1.d[1] // GHASH block 4k+1 - mid - aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 3 - aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 2 - eor $acc_lb, $acc_lb, $t2.16b // GHASH block 4k+1 - low - aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 3 - aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 4 - mov $t6d, $res2.d[1] // GHASH block 4k+2 - mid - aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 3 - eor $t3.8b, $t3.8b, $res1.8b // GHASH block 4k+1 - mid - aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 4 - aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 6 - eor $t6.8b, $t6.8b, $res2.8b // GHASH block 4k+2 - mid - aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 4 - pmull $t3.1q, $t3.1d, $h34k.1d // GHASH block 4k+1 - mid - aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 7 - aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 5 - ins $t6.d[1], $t6.d[0] // GHASH block 4k+2 - mid - aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 5 - aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 8 - aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 5 - aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 6 - eor $acc_mb, $acc_mb, $t3.16b // GHASH block 4k+1 - mid - pmull2 $t4.1q, $res2.2d, $h2.2d // GHASH block 4k+2 - high - pmull $t5.1q, $res2.1d, $h2.1d // GHASH block 4k+2 - low - aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 7 - pmull $t8.1q, $res3.1d, $h1.1d // GHASH block 4k+3 - low - eor $acc_hb, $acc_hb, $t4.16b // GHASH block 4k+2 - high - aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 6 - ldp $input_l1, $input_h1, [$input_ptr, #16] // AES block 4k+5 - load plaintext - aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 8 - mov $t9d, $res3.d[1] // GHASH block 4k+3 - mid - aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 6 - eor $acc_lb, $acc_lb, $t5.16b // GHASH block 4k+2 - low - pmull2 $t6.1q, $t6.2d, $h12k.2d // GHASH block 4k+2 - mid - pmull2 $t7.1q, $res3.2d, $h1.2d // GHASH block 4k+3 - high - eor $t9.8b, $t9.8b, $res3.8b // GHASH block 4k+3 - mid - aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 7 - eor $input_l1, $input_l1, $rkN_l // AES block 4k+5 - round N low - aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 8 - eor $acc_mb, $acc_mb, $t6.16b // GHASH block 4k+2 - mid - aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 7 - eor $input_l2, $input_l2, $rkN_l // AES block 4k+6 - round N low - aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 8 - movi $mod_constant.8b, #0xc2 - pmull $t9.1q, $t9.1d, $h12k.1d // GHASH block 4k+3 - mid - eor $acc_hb, $acc_hb, $t7.16b // GHASH block 4k+3 - high - cmp $rounds, #12 // setup flags for AES-128/192/256 check - fmov $ctr_t1d, $input_l1 // AES block 4k+5 - mov low - ldp $input_l0, $input_h0, [$input_ptr, #0] // AES block 4k+4 - load plaintext - b.lt .Lenc_main_loop_continue // branch if AES-128 - - aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 9 - aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 9 - aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 9 - aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 9 - aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 10 - aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 10 - aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 10 - aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 10 - b.eq .Lenc_main_loop_continue // branch if AES-192 - - aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 11 - aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 11 - aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 11 - aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 11 - aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 12 - aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 12 - aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 12 - aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 12 - + cmp $input_ptr, $main_end_input_ptr // check if we have <= 4 blocks to process in the tail + eor $h34k.16b, $h34k.16b, $acc_h.16b // Karatsuba key: H3^H4 + aese $ctr0.16b, $rkNm1.16b // AES block 0 - round N-1 + aese $ctr1.16b, $rkNm1.16b // AES block 1 - round N-1 + aese $ctr2.16b, $rkNm1.16b // AES block 2 - round N-1 + aese $ctr3.16b, $rkNm1.16b // AES block 3 - round N-1 + trn1 $mod_constant.2d, $h1.2d, $h2.2d // Karatsuba key: H2_high | H1_high + eor $h12k.16b, $h12k.16b, $mod_constant.16b // Karatsuba key: H1^H2 + b.ge .Lenc_tail // handle tail if no more full 4-block sets + ldp $res2q, $res3q, [$input_ptr, #32] // AES blocks 2,3 load plaintext + ldp $res0q, $res1q, [$input_ptr], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + PLATFORM_EOR3($res0, $res0, $v_rkN, $ctr0) // AES block 0 - result = PT ^ AES(ctr0) + PLATFORM_EOR3($res1, $res1, $v_rkN, $ctr1) // AES block 1 - result = PT ^ AES(ctr1) + PLATFORM_EOR3($res2, $res2, $v_rkN, $ctr2) // AES block 2 - result = PT ^ AES(ctr2) + PLATFORM_EOR3($res3, $res3, $v_rkN, $ctr3) // AES block 3 - result = PT ^ AES(ctr3) + st1 { $res0.16b, $res1.16b, $res2.16b, $res3.16b}, [$output_ptr], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp $ctr0q, $ctr1q, [sp, #$ctr0_sp_offset] + ldp $ctr2q, $ctr3q, [sp, #$ctr2_sp_offset] + // Prepare and store counter values for the third iteration + rev $tmp_gpr_w, $W_T32 + str $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}] // ctr + 8 + add $tmp_gpr_w, $W_T32, #1 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}] // ctr + 9 + add $tmp_gpr_w, $W_T32, #2 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}] // ctr + 10 + add $tmp_gpr_w, $W_T32, #3 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}] // ctr + 11 + add $W_T32, $W_T32, #4 // Advance counter base + cmp $input_ptr, $main_end_input_ptr // check if we have <= 4 blocks remaining + b.ge .Lenc_prepretail // go to prepretail if < 2 full loops left +.Lenc_main_loop: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- + aese $ctr0.16b, $rk0.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 0 + aese $ctr1.16b, $rk0.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 0 + aese $ctr2.16b, $rk0.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 0 + aese $ctr3.16b, $rk0.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 0 + ldr $mod_constantd, [sp, #$mod_constant_sp_offset] // Load GHASH reduction constant + aese $ctr0.16b, $rk1.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 1 + aese $ctr1.16b, $rk1.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 1 + aese $ctr2.16b, $rk1.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 1 + aese $ctr3.16b, $rk1.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 $res0.16b, $res0.16b // GHASH block 4k - Byte swap CT + rev64 $res1.16b, $res1.16b // GHASH block 4k+1 - Byte swap CT + rev64 $res2.16b, $res2.16b // GHASH block 4k+2 - Byte swap CT + rev64 $res3.16b, $res3.16b // GHASH block 4k+3 - Byte swap CT + aese $ctr0.16b, $rk2.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 2 + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 // GHASH - prepare acc for XOR + aese $ctr1.16b, $rk2.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 2 + eor $res0.16b, $res0.16b, $acc_l.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} + aese $ctr2.16b, $rk2.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 2 + pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 4k - low + aese $ctr3.16b, $rk2.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 2 + pmull2 $acc_h.1q, $res2.2d, $h2.2d // GHASH block 4k+2 - high + aese $ctr0.16b, $rk3.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 3 + mov $acc_md, $h34k.d[1] // GHASH block 4k - mid Karatsuba key + aese $ctr1.16b, $rk3.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 3 + mov $ghash_t0d, $res0.d[1] // GHASH block 4k - mid + aese $ctr2.16b, $rk3.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 3 + eor $ghash_t0.8b, $ghash_t0.8b, $res0.8b // GHASH block 4k - mid + aese $ctr3.16b, $rk3.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 3 + mov $ghash_t1d, $res1.d[1] // GHASH block 4k+1 - mid + aese $ctr0.16b, $rk4.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 4 + eor $ghash_t1.8b, $ghash_t1.8b, $res1.8b // GHASH block 4k+1 - mid + aese $ctr1.16b, $rk4.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 4 + pmull $acc_m.1q, $ghash_t0.1d, $acc_m.1d // GHASH block 4k - mid + aese $ctr2.16b, $rk4.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 4 + pmull $ghash_t1.1q, $ghash_t1.1d, $h34k.1d // GHASH block 4k+1 - mid + aese $ctr3.16b, $rk4.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 4 + eor $acc_m.16b, $acc_m.16b, $ghash_t1.16b // GHASH block 4k+1 - mid + aese $ctr0.16b, $rk5.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 5 + ext $ghash_t2.16b, $ghash_t2.16b, $res2.16b, #8 // GHASH block 4k+2 - mid + aese $ctr1.16b, $rk5.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 5 + eor $ghash_t2.16b, $ghash_t2.16b, $res2.16b // GHASH block 4k+2 - mid + aese $ctr2.16b, $rk5.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 5 + pmull2 $ghash_t2.1q, $ghash_t2.2d, $h12k.2d // GHASH block 4k+2 - mid + aese $ctr3.16b, $rk5.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 5 + mov $ghash_t3d, $res3.d[1] // GHASH block 4k+3 - mid + aese $ctr0.16b, $rk6.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 6 + eor $ghash_t3.8b, $ghash_t3.8b, $res3.8b // GHASH block 4k+3 - mid + aese $ctr1.16b, $rk6.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 6 + pmull $ghash_t3.1q, $ghash_t3.1d, $h12k.1d // GHASH block 4k+3 - mid + aese $ctr2.16b, $rk6.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 6 + PLATFORM_EOR3($acc_m, $acc_m, $ghash_t2, $ghash_t3) // GHASH block 4k+2/3 - mid + aese $ctr3.16b, $rk6.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 6 + pmull2 $ghash_t2.1q, $res0.2d, $h4.2d // GHASH block 4k - high + aese $ctr0.16b, $rk7.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 7 + pmull2 $ghash_t1.1q, $res3.2d, $h1.2d // GHASH block 4k+3 - high + eor $ghash_t2.16b, $ghash_t2.16b, $ghash_t1.16b // GHASH block 4k+3 - high + aese $ctr1.16b, $rk7.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 7 + pmull2 $ghash_t3.1q, $res1.2d, $h3.2d // GHASH block 4k+1 - high + aese $ctr2.16b, $rk7.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 7 + pmull $ghash_t1.1q, $res2.1d, $h2.1d // GHASH block 4k+2 - low + aese $ctr3.16b, $rk7.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 7 + pmull $ghash_t0.1q, $res1.1d, $h3.1d // GHASH block 4k+1 - low + PLATFORM_EOR3($acc_h, $acc_h, $ghash_t2, $ghash_t3) // GHASH block 4k/1/2/3 - high + pmull $ghash_t2.1q, $res3.1d, $h1.1d // GHASH block 4k+3 - low + ldp $res2q, $res3q, [$input_ptr, #32] + ldp $res0q, $res1q, [$input_ptr], #64 + eor $ghash_t0.16b, $ghash_t0.16b, $ghash_t1.16b // GHASH block 4k+1 - low + aese $ctr0.16b, $rk8.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 8 + PLATFORM_EOR3($acc_l, $acc_l, $ghash_t2, $ghash_t0) // GHASH block 4k/1/2/3 - low + aese $ctr1.16b, $rk8.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 8 + PLATFORM_EOR3($acc_m, $acc_m, $acc_h, $acc_l) // MODULO - karatsuba tidy up + aese $ctr2.16b, $rk8.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 8 + pmull $ghash_t0.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid + aese $ctr3.16b, $rk8.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 8 + cmp $rounds, #12 // setup flags for AES-128/192/256 check + b.lt .Lenc_main_loop_continue // branch if AES-128 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #144] // load rk9, rk10 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 9 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 9 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 9 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 9 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 10 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 10 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 10 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 10 + b.eq .Lenc_main_loop_continue // branch if AES-192 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #176] // load rk11, rk12 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 11 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 11 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 11 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 11 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 12 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 12 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 12 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 12 .Lenc_main_loop_continue: - shl $mod_constantd, $mod_constantd, #56 // mod_constant - eor $acc_lb, $acc_lb, $t8.16b // GHASH block 4k+3 - low - eor $acc_mb, $acc_mb, $t9.16b // GHASH block 4k+3 - mid - add $rctr32w, $rctr32w, #1 // CTR block 4k+3 - eor $t9.16b, $acc_lb, $acc_hb // MODULO - karatsuba tidy up - add $input_ptr, $input_ptr, #64 // AES input_ptr update - pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid - rev $ctr32w, $rctr32w // CTR block 4k+8 - ext $acc_hb, $acc_hb, $acc_hb, #8 // MODULO - other top alignment - eor $input_l0, $input_l0, $rkN_l // AES block 4k+4 - round N low - eor $acc_mb, $acc_mb, $t9.16b // MODULO - karatsuba tidy up - eor $input_h0, $input_h0, $rkN_h // AES block 4k+4 - round N high - fmov $ctr_t0d, $input_l0 // AES block 4k+4 - mov low - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+8 - eor $mod_t.16b, $acc_hb, $mod_t.16b // MODULO - fold into mid - eor $input_h1, $input_h1, $rkN_h // AES block 4k+5 - round N high - eor $input_h3, $input_h3, $rkN_h // AES block 4k+7 - round N high - add $rctr32w, $rctr32w, #1 // CTR block 4k+8 - aese $ctr0b, $rkNm1 // AES block 4k+4 - round N-1 - fmov $ctr_t0.d[1], $input_h0 // AES block 4k+4 - mov high - eor $acc_mb, $acc_mb, $mod_t.16b // MODULO - fold into mid - fmov $ctr_t3d, $input_l3 // AES block 4k+7 - mov low - aese $ctr1b, $rkNm1 // AES block 4k+5 - round N-1 - fmov $ctr_t1.d[1], $input_h1 // AES block 4k+5 - mov high - fmov $ctr_t2d, $input_l2 // AES block 4k+6 - mov low - cmp $input_ptr, $main_end_input_ptr // LOOP CONTROL - fmov $ctr_t2.d[1], $input_h2 // AES block 4k+6 - mov high - pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low - eor $res0b, $ctr_t0b, $ctr0b // AES block 4k+4 - result - fmov $ctr0d, $ctr96_b64x // CTR block 4k+8 - fmov $ctr0.d[1], $ctr32x // CTR block 4k+8 - rev $ctr32w, $rctr32w // CTR block 4k+9 - add $rctr32w, $rctr32w, #1 // CTR block 4k+9 - eor $res1b, $ctr_t1b, $ctr1b // AES block 4k+5 - result - fmov $ctr1d, $ctr96_b64x // CTR block 4k+9 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+9 - fmov $ctr1.d[1], $ctr32x // CTR block 4k+9 - aese $ctr2b, $rkNm1 // AES block 4k+6 - round N-1 - rev $ctr32w, $rctr32w // CTR block 4k+10 - st1 { $res0b}, [$output_ptr], #16 // AES block 4k+4 - store result - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+10 - eor $acc_lb, $acc_lb, $acc_hb // MODULO - fold into low - fmov $ctr_t3.d[1], $input_h3 // AES block 4k+7 - mov high - ext $acc_mb, $acc_mb, $acc_mb, #8 // MODULO - other mid alignment - st1 { $res1b}, [$output_ptr], #16 // AES block 4k+5 - store result - add $rctr32w, $rctr32w, #1 // CTR block 4k+10 - aese $ctr3b, $rkNm1 // AES block 4k+7 - round N-1 - eor $res2b, $ctr_t2b, $ctr2b // AES block 4k+6 - result - fmov $ctr2d, $ctr96_b64x // CTR block 4k+10 - st1 { $res2b}, [$output_ptr], #16 // AES block 4k+6 - store result - fmov $ctr2.d[1], $ctr32x // CTR block 4k+10 - rev $ctr32w, $rctr32w // CTR block 4k+11 - eor $acc_lb, $acc_lb, $acc_mb // MODULO - fold into low - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+11 - eor $res3b, $ctr_t3b, $ctr3b // AES block 4k+7 - result - st1 { $res3b}, [$output_ptr], #16 // AES block 4k+7 - store result - b.lt .Lenc_main_loop - -.Lenc_prepretail: // PREPRETAIL - aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 0 - rev64 $res2b, $res2b // GHASH block 4k+2 (t0, t1, and t2 free) - aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 0 - fmov $ctr3d, $ctr96_b64x // CTR block 4k+3 - aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 0 - rev64 $res0b, $res0b // GHASH block 4k (only t0 is free) - fmov $ctr3.d[1], $ctr32x // CTR block 4k+3 - ext $acc_lb, $acc_lb, $acc_lb, #8 // PRE 0 - aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 1 - aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 1 - eor $res0b, $res0b, $acc_lb // PRE 1 - rev64 $res1b, $res1b // GHASH block 4k+1 (t0 and t1 free) - aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 2 - aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 0 - mov $acc_md, $h34k.d[1] // GHASH block 4k - mid - aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 1 - pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 4k - low - mov $t0d, $res0.d[1] // GHASH block 4k - mid - pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 4k - high - aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 3 - aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 2 - eor $t0.8b, $t0.8b, $res0.8b // GHASH block 4k - mid - aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 2 - aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 1 - aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 3 - pmull $acc_m.1q, $t0.1d, $acc_m.1d // GHASH block 4k - mid - pmull2 $t1.1q, $res1.2d, $h3.2d // GHASH block 4k+1 - high - pmull $t2.1q, $res1.1d, $h3.1d // GHASH block 4k+1 - low - aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 2 - eor $acc_hb, $acc_hb, $t1.16b // GHASH block 4k+1 - high - mov $t3d, $res1.d[1] // GHASH block 4k+1 - mid - aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 3 - eor $acc_lb, $acc_lb, $t2.16b // GHASH block 4k+1 - low - aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 3 - eor $t3.8b, $t3.8b, $res1.8b // GHASH block 4k+1 - mid - mov $t6d, $res2.d[1] // GHASH block 4k+2 - mid - aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 4 - rev64 $res3b, $res3b // GHASH block 4k+3 (t0, t1, t2 and t3 free) - aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 4 - pmull $t3.1q, $t3.1d, $h34k.1d // GHASH block 4k+1 - mid - eor $t6.8b, $t6.8b, $res2.8b // GHASH block 4k+2 - mid - add $rctr32w, $rctr32w, #1 // CTR block 4k+3 - pmull $t5.1q, $res2.1d, $h2.1d // GHASH block 4k+2 - low - aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 5 - aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 4 - eor $acc_mb, $acc_mb, $t3.16b // GHASH block 4k+1 - mid - pmull2 $t4.1q, $res2.2d, $h2.2d // GHASH block 4k+2 - high - eor $acc_lb, $acc_lb, $t5.16b // GHASH block 4k+2 - low - ins $t6.d[1], $t6.d[0] // GHASH block 4k+2 - mid - aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 5 - eor $acc_hb, $acc_hb, $t4.16b // GHASH block 4k+2 - high - mov $t9d, $res3.d[1] // GHASH block 4k+3 - mid - aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 4 - pmull2 $t6.1q, $t6.2d, $h12k.2d // GHASH block 4k+2 - mid - eor $t9.8b, $t9.8b, $res3.8b // GHASH block 4k+3 - mid - pmull2 $t7.1q, $res3.2d, $h1.2d // GHASH block 4k+3 - high - aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 5 - pmull $t9.1q, $t9.1d, $h12k.1d // GHASH block 4k+3 - mid - eor $acc_mb, $acc_mb, $t6.16b // GHASH block 4k+2 - mid - aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 5 - aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 6 - aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 6 - aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 6 - movi $mod_constant.8b, #0xc2 - aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 6 - aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 7 - eor $acc_hb, $acc_hb, $t7.16b // GHASH block 4k+3 - high - aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 7 - aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 7 - shl $mod_constantd, $mod_constantd, #56 // mod_constant - aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 8 - eor $acc_mb, $acc_mb, $t9.16b // GHASH block 4k+3 - mid - pmull $t8.1q, $res3.1d, $h1.1d // GHASH block 4k+3 - low - aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 8 - cmp $rounds, #12 // setup flags for AES-128/192/256 check - aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 8 - eor $acc_lb, $acc_lb, $t8.16b // GHASH block 4k+3 - low - aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 7 - eor $acc_mb, $acc_mb, $acc_hb // karatsuba tidy up - aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 8 - pmull $t1.1q, $acc_h.1d, $mod_constant.1d - ext $acc_hb, $acc_hb, $acc_hb, #8 - eor $acc_mb, $acc_mb, $acc_lb - b.lt .Lenc_finish_prepretail // branch if AES-128 - - aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 9 - aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 9 - aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 9 - aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 9 - aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 10 - aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 10 - aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 10 - aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 10 - b.eq .Lenc_finish_prepretail // branch if AES-192 - - aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 11 - aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 11 - aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 11 - aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 11 - aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 12 - aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 12 - aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 12 - aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 12 - + ext $acc_h.16b, $acc_h.16b, $acc_h.16b, #8 // MODULO - other top alignment + PLATFORM_EOR3($acc_m, $acc_m, $ghash_t0, $acc_h) // MODULO - fold into mid + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low + ext $ghash_t0.16b, $acc_m.16b, $acc_m.16b, #8 // MODULO - other mid alignment + PLATFORM_EOR3($acc_l, $acc_h, $acc_l, $ghash_t0) // MODULO - fold into low + aese $ctr0.16b, $rkNm1.16b // AES block 4k+4 - round N-1 + PLATFORM_EOR3($res0, $res0, $v_rkN, $ctr0) // AES block 4k+4 - result + aese $ctr1.16b, $rkNm1.16b // AES block 4k+5 - round N-1 + PLATFORM_EOR3($res1, $res1, $v_rkN, $ctr1) // AES block 4k+5 - result + aese $ctr2.16b, $rkNm1.16b // AES block 4k+6 - round N-1 + PLATFORM_EOR3($res2, $res2, $v_rkN, $ctr2) // AES block 4k+6 - result + aese $ctr3.16b, $rkNm1.16b // AES block 4k+7 - round N-1 + PLATFORM_EOR3($res3, $res3, $v_rkN, $ctr3) // AES block 4k+7 - result + ldp $ctr0q, $ctr1q, [sp, #$ctr0_sp_offset] + ldp $ctr2q, $ctr3q, [sp, #$ctr2_sp_offset] + // We used these registers as temporaries above so reload the RKs. + ldp $rk2q, $rk3q, [$cc, #32] // load rk2, rk3 + ldp $rk4q, $rk5q, [$cc, #64] // load rk4, rk5 + st1 { $res0.16b, $res1.16b, $res2.16b, $res3.16b}, [$output_ptr], #64 // AES blocks 4k+4-7 - store result + rev $tmp_gpr_w, $W_T32 + str $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}] + add $tmp_gpr_w, $W_T32, #1 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}] + add $tmp_gpr_w, $W_T32, #2 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}] + add $tmp_gpr_w, $W_T32, #3 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}] + add $W_T32, $W_T32, #4 + cmp $input_ptr, $main_end_input_ptr // .LOOP CONTROL + b.lt .Lenc_main_loop +.Lenc_prepretail: // PREPRETAIL + aese $ctr1.16b, $rk0.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 0 + rev64 $res2.16b, $res2.16b // GHASH block 2 + aese $ctr2.16b, $rk0.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 0 + aese $ctr0.16b, $rk0.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 0 + rev64 $res0.16b, $res0.16b // GHASH block 0 + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 // PRE 0 + aese $ctr2.16b, $rk1.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 1 + aese $ctr0.16b, $rk1.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 1 + eor $res0.16b, $res0.16b, $acc_l.16b // PRE 1 + rev64 $res1.16b, $res1.16b // GHASH block 1 + aese $ctr2.16b, $rk2.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 2 + aese $ctr3.16b, $rk0.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 0 + mov $acc_md, $h34k.d[1] // GHASH block 0 - mid Karatsuba key + aese $ctr1.16b, $rk1.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 1 + pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 0 - low + mov $mod_constantd, $res0.d[1] // GHASH block 0 - mid + pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 0 - high + aese $ctr2.16b, $rk3.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 3 + aese $ctr1.16b, $rk2.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 2 + eor $mod_constant.8b, $mod_constant.8b, $res0.8b // GHASH block 0 - mid + aese $ctr0.16b, $rk2.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 2 + aese $ctr3.16b, $rk1.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 1 + aese $ctr1.16b, $rk3.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 3 + pmull $acc_m.1q, $mod_constant.1d, $acc_m.1d // GHASH block 0 - mid + pmull2 $res0.1q, $res1.2d, $h3.2d // GHASH block 1 - high + pmull $mod_constant.1q, $res1.1d, $h3.1d // GHASH block 1 - low + aese $ctr3.16b, $rk2.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 2 + eor $acc_h.16b, $acc_h.16b, $res0.16b // GHASH block 1 - high + mov $res0d, $res1.d[1] // GHASH block 1 - mid + aese $ctr0.16b, $rk3.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 3 + eor $acc_l.16b, $acc_l.16b, $mod_constant.16b // GHASH block 1 - low + aese $ctr3.16b, $rk3.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 3 + eor $res0.8b, $res0.8b, $res1.8b // GHASH block 1 - mid + mov $mod_constantd, $res2.d[1] // GHASH block 2 - mid + aese $ctr0.16b, $rk4.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 4 + rev64 $res3.16b, $res3.16b // GHASH block 3 + aese $ctr3.16b, $rk4.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 4 + pmull $res0.1q, $res0.1d, $h34k.1d // GHASH block 1 - mid + eor $mod_constant.8b, $mod_constant.8b, $res2.8b // GHASH block 2 - mid + pmull $res1.1q, $res2.1d, $h2.1d // GHASH block 2 - low + aese $ctr3.16b, $rk5.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 5 + aese $ctr2.16b, $rk4.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 4 + eor $acc_m.16b, $acc_m.16b, $res0.16b // GHASH block 1 - mid + pmull2 $res0.1q, $res2.2d, $h2.2d // GHASH block 2 - high + eor $acc_l.16b, $acc_l.16b, $res1.16b // GHASH block 2 - low + ins $mod_constant.d[1], $mod_constant.d[0] // GHASH block 2 - mid + aese $ctr2.16b, $rk5.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 5 + eor $acc_h.16b, $acc_h.16b, $res0.16b // GHASH block 2 - high + mov $res0d, $res3.d[1] // GHASH block 3 - mid + aese $ctr1.16b, $rk4.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 4 + pmull2 $mod_constant.1q, $mod_constant.2d, $h12k.2d // GHASH block 2 - mid + eor $res0.8b, $res0.8b, $res3.8b // GHASH block 3 - mid + pmull2 $res1.1q, $res3.2d, $h1.2d // GHASH block 3 - high + aese $ctr1.16b, $rk5.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 5 + pmull $res0.1q, $res0.1d, $h12k.1d // GHASH block 3 - mid + eor $acc_m.16b, $acc_m.16b, $mod_constant.16b // GHASH block 2 - mid + aese $ctr0.16b, $rk5.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 5 + aese $ctr1.16b, $rk6.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 6 + aese $ctr2.16b, $rk6.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 6 + aese $ctr0.16b, $rk6.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 6 + aese $ctr3.16b, $rk6.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 6 + aese $ctr1.16b, $rk7.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 7 + eor $acc_h.16b, $acc_h.16b, $res1.16b // GHASH block 3 - high + aese $ctr0.16b, $rk7.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 7 + aese $ctr3.16b, $rk7.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 7 + ldr $mod_constantd, [sp, #$mod_constant_sp_offset] + aese $ctr1.16b, $rk8.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 8 + eor $acc_m.16b, $acc_m.16b, $res0.16b // GHASH block 3 - mid + pmull $res2.1q, $res3.1d, $h1.1d // GHASH block 3 - low + aese $ctr3.16b, $rk8.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 8 + cmp $rounds, #12 // setup flags for AES-128/192/256 check + aese $ctr0.16b, $rk8.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 8 + eor $acc_l.16b, $acc_l.16b, $res2.16b // GHASH block 3 - low + aese $ctr2.16b, $rk7.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 7 + eor $acc_m.16b, $acc_m.16b, $acc_h.16b // karatsuba tidy up + aese $ctr2.16b, $rk8.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 8 + pmull $res0.1q, $acc_h.1d, $mod_constant.1d + ext $acc_h.16b, $acc_h.16b, $acc_h.16b, #8 + eor $acc_m.16b, $acc_m.16b, $acc_l.16b + b.lt .Lenc_finish_prepretail // branch if AES-128 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #144] // load rk9, rk10 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 9 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 9 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 9 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 9 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 10 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 10 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 10 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 10 + b.eq .Lenc_finish_prepretail // branch if AES-192 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #176] // load rk11, rk12 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 11 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 11 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 11 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 11 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 12 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 12 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 12 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 12 .Lenc_finish_prepretail: - eor $acc_mb, $acc_mb, $t1.16b - eor $acc_mb, $acc_mb, $acc_hb - pmull $t1.1q, $acc_m.1d, $mod_constant.1d - ext $acc_mb, $acc_mb, $acc_mb, #8 - aese $ctr1b, $rkNm1 // AES block 4k+5 - round N-1 - eor $acc_lb, $acc_lb, $t1.16b - aese $ctr3b, $rkNm1 // AES block 4k+7 - round N-1 - aese $ctr0b, $rkNm1 // AES block 4k+4 - round N-1 - aese $ctr2b, $rkNm1 // AES block 4k+6 - round N-1 - eor $acc_lb, $acc_lb, $acc_mb - -.Lenc_tail: // TAIL - ext $t0.16b, $acc_lb, $acc_lb, #8 // prepare final partial tag - sub $main_end_input_ptr, $end_input_ptr, $input_ptr // main_end_input_ptr is number of bytes left to process - ldp $input_l0, $input_h0, [$input_ptr], #16 // AES block 4k+4 - load plaintext - eor $input_l0, $input_l0, $rkN_l // AES block 4k+4 - round N low - eor $input_h0, $input_h0, $rkN_h // AES block 4k+4 - round N high - cmp $main_end_input_ptr, #48 - fmov $ctr_t0d, $input_l0 // AES block 4k+4 - mov low - fmov $ctr_t0.d[1], $input_h0 // AES block 4k+4 - mov high - eor $res1b, $ctr_t0b, $ctr0b // AES block 4k+4 - result - b.gt .Lenc_blocks_more_than_3 - cmp $main_end_input_ptr, #32 - mov $ctr3b, $ctr2b - movi $acc_l.8b, #0 - movi $acc_h.8b, #0 - sub $rctr32w, $rctr32w, #1 - mov $ctr2b, $ctr1b - movi $acc_m.8b, #0 - b.gt .Lenc_blocks_more_than_2 - mov $ctr3b, $ctr1b - sub $rctr32w, $rctr32w, #1 - cmp $main_end_input_ptr, #16 - b.gt .Lenc_blocks_more_than_1 - sub $rctr32w, $rctr32w, #1 - b .Lenc_blocks_less_than_1 -.Lenc_blocks_more_than_3: // blocks left > 3 - st1 { $res1b}, [$output_ptr], #16 // AES final-3 block - store result - ldp $input_l0, $input_h0, [$input_ptr], #16 // AES final-2 block - load input low & high - rev64 $res0b, $res1b // GHASH final-3 block - eor $input_l0, $input_l0, $rkN_l // AES final-2 block - round N low - eor $res0b, $res0b, $t0.16b // feed in partial tag - eor $input_h0, $input_h0, $rkN_h // AES final-2 block - round N high - mov $rk4d, $res0.d[1] // GHASH final-3 block - mid - fmov $res1d, $input_l0 // AES final-2 block - mov low - fmov $res1.d[1], $input_h0 // AES final-2 block - mov high - eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-3 block - mid - movi $t0.8b, #0 // suppress further partial tag feed in - mov $acc_md, $h34k.d[1] // GHASH final-3 block - mid - pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH final-3 block - low - pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH final-3 block - high - pmull $acc_m.1q, $rk4v.1d, $acc_m.1d // GHASH final-3 block - mid - eor $res1b, $res1b, $ctr1b // AES final-2 block - result -.Lenc_blocks_more_than_2: // blocks left > 2 - st1 { $res1b}, [$output_ptr], #16 // AES final-2 block - store result - ldp $input_l0, $input_h0, [$input_ptr], #16 // AES final-1 block - load input low & high - rev64 $res0b, $res1b // GHASH final-2 block - eor $input_l0, $input_l0, $rkN_l // AES final-1 block - round N low - eor $res0b, $res0b, $t0.16b // feed in partial tag - fmov $res1d, $input_l0 // AES final-1 block - mov low - eor $input_h0, $input_h0, $rkN_h // AES final-1 block - round N high - fmov $res1.d[1], $input_h0 // AES final-1 block - mov high - movi $t0.8b, #0 // suppress further partial tag feed in - pmull2 $rk2q1, $res0.2d, $h3.2d // GHASH final-2 block - high - mov $rk4d, $res0.d[1] // GHASH final-2 block - mid - pmull $rk3q1, $res0.1d, $h3.1d // GHASH final-2 block - low - eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-2 block - mid - eor $res1b, $res1b, $ctr2b // AES final-1 block - result - eor $acc_hb, $acc_hb, $rk2 // GHASH final-2 block - high - pmull $rk4v.1q, $rk4v.1d, $h34k.1d // GHASH final-2 block - mid - eor $acc_lb, $acc_lb, $rk3 // GHASH final-2 block - low - eor $acc_mb, $acc_mb, $rk4v.16b // GHASH final-2 block - mid -.Lenc_blocks_more_than_1: // blocks left > 1 - st1 { $res1b}, [$output_ptr], #16 // AES final-1 block - store result - rev64 $res0b, $res1b // GHASH final-1 block - ldp $input_l0, $input_h0, [$input_ptr], #16 // AES final block - load input low & high - eor $res0b, $res0b, $t0.16b // feed in partial tag - movi $t0.8b, #0 // suppress further partial tag feed in - eor $input_l0, $input_l0, $rkN_l // AES final block - round N low - mov $rk4d, $res0.d[1] // GHASH final-1 block - mid - pmull2 $rk2q1, $res0.2d, $h2.2d // GHASH final-1 block - high - eor $input_h0, $input_h0, $rkN_h // AES final block - round N high - eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-1 block - mid - eor $acc_hb, $acc_hb, $rk2 // GHASH final-1 block - high - ins $rk4v.d[1], $rk4v.d[0] // GHASH final-1 block - mid - fmov $res1d, $input_l0 // AES final block - mov low - fmov $res1.d[1], $input_h0 // AES final block - mov high - pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d // GHASH final-1 block - mid - pmull $rk3q1, $res0.1d, $h2.1d // GHASH final-1 block - low - eor $res1b, $res1b, $ctr3b // AES final block - result - eor $acc_mb, $acc_mb, $rk4v.16b // GHASH final-1 block - mid - eor $acc_lb, $acc_lb, $rk3 // GHASH final-1 block - low -.Lenc_blocks_less_than_1: // blocks left <= 1 - and $bit_length, $bit_length, #127 // bit_length %= 128 - mvn $rkN_l, xzr // rkN_l = 0xffffffffffffffff - sub $bit_length, $bit_length, #128 // bit_length -= 128 - neg $bit_length, $bit_length // bit_length = 128 - #bits in input (in range [1,128]) - ld1 { $rk0}, [$output_ptr] // load existing bytes where the possibly partial last block is to be stored - mvn $rkN_h, xzr // rkN_h = 0xffffffffffffffff - and $bit_length, $bit_length, #127 // bit_length %= 128 - lsr $rkN_h, $rkN_h, $bit_length // rkN_h is mask for top 64b of last block - cmp $bit_length, #64 - csel $input_l0, $rkN_l, $rkN_h, lt - csel $input_h0, $rkN_h, xzr, lt - fmov $ctr0d, $input_l0 // ctr0b is mask for last block - fmov $ctr0.d[1], $input_h0 - and $res1b, $res1b, $ctr0b // possibly partial last block has zeroes in highest bits - rev64 $res0b, $res1b // GHASH final block - eor $res0b, $res0b, $t0.16b // feed in partial tag - bif $res1b, $rk0, $ctr0b // insert existing bytes in top end of result before storing - pmull2 $rk2q1, $res0.2d, $h1.2d // GHASH final block - high - mov $t0d, $res0.d[1] // GHASH final block - mid - rev $ctr32w, $rctr32w - pmull $rk3q1, $res0.1d, $h1.1d // GHASH final block - low - eor $acc_hb, $acc_hb, $rk2 // GHASH final block - high - eor $t0.8b, $t0.8b, $res0.8b // GHASH final block - mid - pmull $t0.1q, $t0.1d, $h12k.1d // GHASH final block - mid - eor $acc_lb, $acc_lb, $rk3 // GHASH final block - low - eor $acc_mb, $acc_mb, $t0.16b // GHASH final block - mid - movi $mod_constant.8b, #0xc2 - eor $t9.16b, $acc_lb, $acc_hb // MODULO - karatsuba tidy up - shl $mod_constantd, $mod_constantd, #56 // mod_constant - eor $acc_mb, $acc_mb, $t9.16b // MODULO - karatsuba tidy up - pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid - ext $acc_hb, $acc_hb, $acc_hb, #8 // MODULO - other top alignment - eor $acc_mb, $acc_mb, $mod_t.16b // MODULO - fold into mid - eor $acc_mb, $acc_mb, $acc_hb // MODULO - fold into mid - pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low - ext $acc_mb, $acc_mb, $acc_mb, #8 // MODULO - other mid alignment - str $ctr32w, [$counter, #12] // store the updated counter - st1 { $res1b}, [$output_ptr] // store all 16B - eor $acc_lb, $acc_lb, $acc_hb // MODULO - fold into low - eor $acc_lb, $acc_lb, $acc_mb // MODULO - fold into low - ext $acc_lb, $acc_lb, $acc_lb, #8 - rev64 $acc_lb, $acc_lb - mov x0, $len - st1 { $acc_l.16b }, [$current_tag] - ldp x19, x20, [sp, #16] - ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] - ldp d8, d9, [sp, #64] - ldp d10, d11, [sp, #80] - ldp d12, d13, [sp, #96] - ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 - AARCH64_VALIDATE_LINK_REGISTER - ret + aese $ctr0.16b, $rkNm1.16b // AES block 0 - round N-1 + aese $ctr1.16b, $rkNm1.16b // AES block 1 - round N-1 + aese $ctr2.16b, $rkNm1.16b // AES block 2 - round N-1 + aese $ctr3.16b, $rkNm1.16b // AES block 3 - round N-1 + PLATFORM_EOR3($acc_m, $acc_m, $res0, $acc_h) + pmull $res0.1q, $acc_m.1d, $mod_constant.1d + ext $acc_m.16b, $acc_m.16b, $acc_m.16b, #8 + PLATFORM_EOR3($acc_l, $acc_l, $res0, $acc_m) +.Lenc_tail: // TAIL: Process remaining 0 to 3 blocks + ext $mod_constant.16b, $acc_l.16b, $acc_l.16b, #8 // Save current GHASH state for partial tag feed-in + sub $main_end_input_ptr, $end_input_ptr, $input_ptr // main_end_input_ptr is number of bytes left to process + ldp $input_l0, $input_h0, [$input_ptr], #16 // AES block 0 - load plaintext + eor $input_l0, $input_l0, $rkN_l // AES block 0 - round N low + eor $input_h0, $input_h0, $rkN_h // AES block 0 - round N high + cmp $main_end_input_ptr, #48 + fmov $res0d, $input_l0 // AES block 0 - mov low + fmov $res0.d[1], $input_h0 // AES block 0 - mov high + eor $res1.16b, $res0.16b, $ctr0.16b // AES block 0 - result + b.gt .Lenc_blocks_more_than_3 + cmp $main_end_input_ptr, #32 + mov $ctr3.16b, $ctr2.16b + movi $acc_l.8b, #0 + movi $acc_h.8b, #0 + mov $ctr2.16b, $ctr1.16b + movi $acc_m.8b, #0 + b.gt .Lenc_blocks_more_than_2 + mov $ctr3.16b, $ctr1.16b + cmp $main_end_input_ptr, #16 + b.gt .Lenc_blocks_more_than_1 + b .Lenc_blocks_less_than_1 +.Lenc_blocks_more_than_3: // blocks left > 3 + st1 { $res1.16b}, [$output_ptr], #16 // AES final-2 block - store result + ldp $input_l0, $input_h0, [$input_ptr], #16 // AES final-2 block - load input low & high + rev64 $res0.16b, $res1.16b // GHASH final-3 block + eor $input_l0, $input_l0, $rkN_l // AES final-2 block - round N low + eor $res0.16b, $res0.16b, $mod_constant.16b // feed in partial tag + eor $input_h0, $input_h0, $rkN_h // AES final-2 block - round N high + mov $ghash_t2d, $res0.d[1] // GHASH final-3 block - mid + fmov $res1d, $input_l0 // AES final-2 block - mov low + fmov $res1.d[1], $input_h0 // AES final-2 block - mov high + eor $ghash_t2.8b, $ghash_t2.8b, $res0.8b // GHASH final-3 block - mid + movi $mod_constant.8b, #0 // suppress further partial tag feed in + mov $acc_md, $h34k.d[1] // GHASH final-3 block - mid + pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH final-3 block - low + pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH final-3 block - high + pmull $acc_m.1q, $ghash_t2.1d, $acc_m.1d // GHASH final-3 block - mid + eor $res1.16b, $res1.16b, $ctr1.16b // AES final-2 block - result +.Lenc_blocks_more_than_2: // blocks left > 2 + st1 { $res1.16b}, [$output_ptr], #16 // AES final-2 block - store result + ldp $input_l0, $input_h0, [$input_ptr], #16 // AES final-1 block - load input low & high + rev64 $res0.16b, $res1.16b // GHASH final-2 block + eor $input_l0, $input_l0, $rkN_l // AES final-1 block - round N low + eor $res0.16b, $res0.16b, $mod_constant.16b // feed in partial tag + fmov $res1d, $input_l0 // AES final-1 block - mov low + eor $input_h0, $input_h0, $rkN_h // AES final-1 block - round N high + fmov $res1.d[1], $input_h0 // AES final-1 block - mov high + movi $mod_constant.8b, #0 // suppress further partial tag feed in + pmull2 $ghash_t0.1q, $res0.2d, $h3.2d // GHASH final-2 block - high + mov $ghash_t2d, $res0.d[1] // GHASH final-2 block - mid + pmull $ghash_t1.1q, $res0.1d, $h3.1d // GHASH final-2 block - low + eor $ghash_t2.8b, $ghash_t2.8b, $res0.8b // GHASH final-2 block - mid + eor $res1.16b, $res1.16b, $ctr2.16b // AES final-1 block - result + eor $acc_h.16b, $acc_h.16b, $ghash_t0.16b // GHASH final-2 block - high + pmull $ghash_t2.1q, $ghash_t2.1d, $h34k.1d // GHASH final-2 block - mid + eor $acc_l.16b, $acc_l.16b, $ghash_t1.16b // GHASH final-2 block - low + eor $acc_m.16b, $acc_m.16b, $ghash_t2.16b // GHASH final-2 block - mid +.Lenc_blocks_more_than_1: // blocks left > 1 + st1 { $res1.16b}, [$output_ptr], #16 // AES final-1 block - store result + rev64 $res0.16b, $res1.16b // GHASH final-1 block: Byte Swap CT + ldp $input_l0, $input_h0, [$input_ptr], #16 // AES final block - load plaintext + eor $res0.16b, $res0.16b, $mod_constant.16b // Feed in partial tag + movi $mod_constant.8b, #0 // Clear for next block + eor $input_l0, $input_l0, $rkN_l // AES final block - round N low + mov $ghash_t2d, $res0.d[1] // GHASH final-1 block - mid + pmull2 $ghash_t0.1q, $res0.2d, $h2.2d // GHASH final-1 block - high + eor $input_h0, $input_h0, $rkN_h // AES final block - round N high + eor $ghash_t2.8b, $ghash_t2.8b, $res0.8b // GHASH final-1 block - mid + eor $acc_h.16b, $acc_h.16b, $ghash_t0.16b // GHASH final-1 block - high + ins $ghash_t2.d[1], $ghash_t2.d[0] // GHASH final-1 block - mid + fmov $res1d, $input_l0 // AES final block - mov low + fmov $res1.d[1], $input_h0 // AES final block - mov high + pmull2 $ghash_t2.1q, $ghash_t2.2d, $h12k.2d // GHASH final-1 block - mid + pmull $ghash_t1.1q, $res0.1d, $h2.1d // GHASH final-1 block - low + eor $res1.16b, $res1.16b, $ctr3.16b // AES final block - result + eor $acc_m.16b, $acc_m.16b, $ghash_t2.16b // GHASH final-1 block - mid + eor $acc_l.16b, $acc_l.16b, $ghash_t1.16b // GHASH final-1 block - low +.Lenc_blocks_less_than_1: // Last partial block handling + add $T32, $T32, $bit_length, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev $W32, $W32 + str $W32, [$counter, #12] // store the updated counter + and $bit_length, $bit_length, #127 // bit_length %= 128 + mvn $rkN_l, xzr // Mask for low 64 bits + sub $bit_length, $bit_length, #128 // + neg $bit_length, $bit_length // Valid bits in the last block (1-128) + ldr $final_block_destq, [$output_ptr] // Load destination for merging + mvn $rkN_h, xzr // Mask for high 64 bits + and $bit_length, $bit_length, #127 // bit_length %= 128 + lsr $rkN_h, $rkN_h, $bit_length // rkN_h is mask for top 64b of last block + cmp $bit_length, #64 + csel $input_l0, $rkN_l, $rkN_h, lt + csel $input_h0, $rkN_h, xzr, lt + fmov $ctr0d, $input_l0 // ctr0d is mask for last block + fmov $ctr0.d[1], $input_h0 + and $res1.16b, $res1.16b, $ctr0.16b // Mask out unused bits of the last CT block + rev64 $res0.16b, $res1.16b // GHASH final block - byte swap + eor $res0.16b, $res0.16b, $mod_constant.16b // Feed in partial tag + bif $res1.16b, $final_block_dest.16b, $ctr0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 $ghash_t0.1q, $res0.2d, $h1.2d // GHASH final block - high + mov $mod_constantd, $res0.d[1] // GHASH final block - mid + pmull $ghash_t1.1q, $res0.1d, $h1.1d // GHASH final block - low + eor $acc_h.16b, $acc_h.16b, $ghash_t0.16b // GHASH final block - high + eor $mod_constant.8b, $mod_constant.8b, $res0.8b // GHASH final block - mid + pmull $mod_constant.1q, $mod_constant.1d, $h12k.1d // GHASH final block - mid + eor $acc_l.16b, $acc_l.16b, $ghash_t1.16b // GHASH final block - low + eor $acc_m.16b, $acc_m.16b, $mod_constant.16b // GHASH final block - mid + eor $res0.16b, $acc_l.16b, $acc_h.16b // MODULO - karatsuba tidy up + fmov $mod_constantd, $mod_constantx + eor $acc_m.16b, $acc_m.16b, $res0.16b // MODULO - karatsuba tidy up + pmull $res3.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid + ext $acc_h.16b, $acc_h.16b, $acc_h.16b, #8 // MODULO - other top alignment + PLATFORM_EOR3($acc_m, $acc_m, $res3, $acc_h) // MODULO - fold into mid + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low + ext $acc_m.16b, $acc_m.16b, $acc_m.16b, #8 // MODULO - other mid alignment + st1 { $res1.16b}, [$output_ptr] // store all 16B + PLATFORM_EOR3($acc_l, $acc_l, $acc_h, $acc_m) // MODULO - fold into low + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 // Byte swap GHASH result + rev64 $acc_l.16b, $acc_l.16b // Final Tag + mov $input_ptr, $len + st1 { $acc_l.16b }, [$current_tag] // Store final tag + ldp x19, x20, [sp, #16] + ldp $mod_constantx, x22, [sp, #32] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret .size aes_gcm_enc_kernel,.-aes_gcm_enc_kernel ___ - -{ -my $t8="v4"; -my $t8d="d4"; -my $t9="v6"; -my $t9d="d6"; -################################################################################ -# size_t aes_gcm_dec_kernel(const uint8_t *in, -# size_t len_bits, -# uint8_t *out, -# u64 *Xi, -# uint8_t ivec[16], -# const void *key); -# -$code.=<<___; + ################################################################################ + # aes_gcm_dec_kernel + ################################################################################ + $code_template .= <<"___"; .global aes_gcm_dec_kernel .type aes_gcm_dec_kernel,%function .align 4 aes_gcm_dec_kernel: - AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! - mov x29, sp - stp x19, x20, [sp, #16] - mov $counter, x4 - mov $cc, x5 - stp x21, x22, [sp, #32] - stp x23, x24, [sp, #48] - stp d8, d9, [sp, #64] - stp d10, d11, [sp, #80] - stp d12, d13, [sp, #96] - stp d14, d15, [sp, #112] - ldr $roundsw, [$cc, #240] - add $input_l1, $cc, $rounds, lsl #4 // borrow input_l1 for last key - ldp $rkN_l, $rkN_h, [$input_l1] // load round N keys - ldr $rkNm1q, [$input_l1, #-16] // load round N-1 keys - lsr $main_end_input_ptr, $bit_length, #3 // byte_len - mov $len, $main_end_input_ptr - ldp $ctr96_b64x, $ctr96_t32x, [$counter] // ctr96_b64, ctr96_t32 - ldr $rk8q, [$cc, #128] // load rk8 - sub $main_end_input_ptr, $main_end_input_ptr, #1 // byte_len - 1 - ldr $rk7q, [$cc, #112] // load rk7 - and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - add $end_input_ptr, $input_ptr, $bit_length, lsr #3 // end_input_ptr - ldr $rk6q, [$cc, #96] // load rk6 - lsr $rctr32x, $ctr96_t32x, #32 - ldr $rk5q, [$cc, #80] // load rk5 - orr $ctr96_t32w, $ctr96_t32w, $ctr96_t32w - ldr $rk3q, [$cc, #48] // load rk3 - add $main_end_input_ptr, $main_end_input_ptr, $input_ptr - rev $rctr32w, $rctr32w // rev_ctr32 - add $rctr32w, $rctr32w, #1 // increment rev_ctr32 - fmov $ctr3d, $ctr96_b64x // CTR block 3 - rev $ctr32w, $rctr32w // CTR block 1 - add $rctr32w, $rctr32w, #1 // CTR block 1 - fmov $ctr1d, $ctr96_b64x // CTR block 1 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 1 - ld1 { $ctr0b}, [$counter] // special case vector load initial counter so we can start first AES block as quickly as possible - fmov $ctr1.d[1], $ctr32x // CTR block 1 - rev $ctr32w, $rctr32w // CTR block 2 - add $rctr32w, $rctr32w, #1 // CTR block 2 - fmov $ctr2d, $ctr96_b64x // CTR block 2 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 2 - fmov $ctr2.d[1], $ctr32x // CTR block 2 - rev $ctr32w, $rctr32w // CTR block 3 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 3 - ldr $rk0q, [$cc, #0] // load rk0 - fmov $ctr3.d[1], $ctr32x // CTR block 3 - add $rctr32w, $rctr32w, #1 // CTR block 3 - ldr $rk4q, [$cc, #64] // load rk4 - ldr $rk1q, [$cc, #16] // load rk1 - aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 0 - ldr $h3q, [$Htable, #48] // load h3l | h3h - ext $h3b, $h3b, $h3b, #8 - aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 0 - ldr $h4q, [$Htable, #80] // load h4l | h4h - ext $h4b, $h4b, $h4b, #8 - aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 0 - ldr $h2q, [$Htable, #32] // load h2l | h2h - ext $h2b, $h2b, $h2b, #8 - aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 0 - ldr $rk2q, [$cc, #32] // load rk2 - aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 1 - aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 1 - ld1 { $acc_lb}, [$current_tag] - ext $acc_lb, $acc_lb, $acc_lb, #8 - rev64 $acc_lb, $acc_lb - aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 1 - ldr $rk9q, [$cc, #144] // load rk9 - aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 1 - ldr $rk12q, [$cc, #192] // load rk12 - aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 2 - ldr $h1q, [$Htable] // load h1l | h1h - ext $h1b, $h1b, $h1b, #8 - aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 2 - ldr $rk10q, [$cc, #160] // load rk10 - aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 2 - aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 3 - aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 2 - aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 3 - aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 4 - aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 3 - aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 3 - aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 4 - aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 4 - aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 4 - aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 5 - aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 5 - aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 5 - aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 5 - aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 6 - aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 6 - cmp $rounds, #12 // setup flags for AES-128/192/256 check - aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 6 - aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 6 - aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 7 - aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 7 - aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 7 - aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 8 - aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 7 - aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 8 - aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 8 - ldr $rk11q, [$cc, #176] // load rk11 - aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 8 - b.lt .Ldec_finish_first_blocks // branch if AES-128 - - aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 9 - aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 9 - aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 9 - aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 9 - aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 10 - aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 10 - aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 10 - aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 10 - b.eq .Ldec_finish_first_blocks // branch if AES-192 - - aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 11 - aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 11 - aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 11 - aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 11 - aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b // AES block 1 - round 12 - aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b // AES block 0 - round 12 - aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b // AES block 2 - round 12 - aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b // AES block 3 - round 12 - + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + stp x19, x20, [sp, #16] + ld1 { $ctr0.16b}, [x4] + mov $ctr1.16b, $ctr0.16b + mov $ctr2.16b, $ctr0.16b + mov $ctr3.16b, $ctr0.16b + mov $counter, x4 + mov $cc, x5 + stp $mod_constantx, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp d8, d9, [sp, #64] + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr $rounds_w, [$cc, #240] // Load number of AES rounds + add $input_l1, $cc, $rounds, lsl #4 // borrow input_l1 for last key + ldp $rkN_l, $rkN_h, [$input_l1] // load round N keys + ldr $rkNm1q, [$input_l1, #-16] // load round N-1 keys + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 // end_input_ptr + lsr $main_end_input_ptr, $bit_length, #3 // byte_len + mov $len, $main_end_input_ptr + ldr $ctr32w, [$counter, #12] // Load scalar 32-bit counter (CTR) + sub $main_end_input_ptr, $main_end_input_ptr, #1 // byte_len - 1 + ldr $rk0q, [$cc, #0] // load rk0 + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + rev $ctr32w, $ctr32w // Reverse it once for big-endian incrementing + uxtw $T32, $ctr32w // Zero extend reversed w9 into x10 + str $ctr0q, [sp, #$ctr0_sp_offset] + str $ctr0q, [sp, #$ctr1_sp_offset] + str $ctr0q, [sp, #$ctr2_sp_offset] + str $ctr0q, [sp, #$ctr3_sp_offset] + rev $tmp_gpr_w, $ctr32w + mov $ctr0.s[3], $tmp_gpr_w + add $tmp_gpr_w, $ctr32w, #1 + rev $tmp_gpr_w, $tmp_gpr_w + mov $ctr1.s[3], $tmp_gpr_w + add $tmp_gpr_w, $ctr32w, #2 + rev $tmp_gpr_w, $tmp_gpr_w + mov $ctr2.s[3], $tmp_gpr_w + add $tmp_gpr_w, $ctr32w, #3 + rev $tmp_gpr_w, $tmp_gpr_w + mov $ctr3.s[3], $tmp_gpr_w + add $tmp_gpr_w, $ctr32w, #4 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #5 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #6 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #7 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}] + add $ctr32w, $ctr32w, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov $mod_constantx, #0xc200000000000000 + str $mod_constantx, [sp, #$mod_constant_sp_offset] + aese $ctr0.16b, $rk0.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 0 + ldp $rk1q, $rk2q, [$cc, #16] // load rk1, rk2 + aese $ctr1.16b, $rk0.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 0 + ldp $rk3q, $rk4q, [$cc, #48] // load rk3, rk4 + aese $ctr2.16b, $rk0.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 0 + ldp $rk5q, $rk6q, [$cc, #80] // load rk5, rk6 + aese $ctr3.16b, $rk0.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 0 + ldp $h2q, $h3q, [$Htable, #32] // load h2, h3 + aese $ctr0.16b, $rk1.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 1 + ldp $rk7q, $rk8q, [$cc, #112] // load rk7, rk8 + aese $ctr1.16b, $rk1.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 1 + aese $ctr2.16b, $rk1.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 1 + aese $ctr3.16b, $rk1.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 1 + aese $ctr0.16b, $rk2.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 2 + ext $h3.16b, $h3.16b, $h3.16b, #8 + aese $ctr1.16b, $rk2.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 2 + ext $h2.16b, $h2.16b, $h2.16b, #8 + ldr $h4q, [$Htable, #80] // load h4 + aese $ctr2.16b, $rk2.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 2 + aese $ctr3.16b, $rk2.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 2 + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr0.16b, $rk3.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 3 + ld1 { $acc_l.16b}, [$current_tag] + aese $ctr1.16b, $rk3.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 3 + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 + aese $ctr2.16b, $rk3.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 3 + rev64 $acc_l.16b, $acc_l.16b + aese $ctr3.16b, $rk3.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 3 + trn2 $h34k.2d, $h3.2d, $h4.2d // h4l | h3l + aese $ctr0.16b, $rk4.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 4 + ldr $h1q, [$Htable] // load h1 + aese $ctr1.16b, $rk4.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 4 + ext $h1.16b, $h1.16b, $h1.16b, #8 + aese $ctr2.16b, $rk4.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 4 + trn1 $acc_h.2d, $h3.2d, $h4.2d // h4h | h3h + aese $ctr3.16b, $rk4.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 4 + trn2 $h12k.2d, $h1.2d, $h2.2d // h2l | h1l + aese $ctr0.16b, $rk5.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 5 + aese $ctr1.16b, $rk5.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 5 + aese $ctr3.16b, $rk5.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 5 + aese $ctr2.16b, $rk5.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 5 + aese $ctr0.16b, $rk6.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 6 + aese $ctr1.16b, $rk6.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 6 + aese $ctr2.16b, $rk6.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 6 + aese $ctr3.16b, $rk6.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 6 + aese $ctr0.16b, $rk7.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 7 + aese $ctr1.16b, $rk7.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 7 + aese $ctr2.16b, $rk7.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 7 + aese $ctr3.16b, $rk7.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 7 + aese $ctr0.16b, $rk8.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 8 + aese $ctr1.16b, $rk8.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 8 + aese $ctr2.16b, $rk8.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 8 + aese $ctr3.16b, $rk8.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 8 + cmp $rounds, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_finish_first_blocks // branch if AES-128 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #144] // load rk9, rk10 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 9 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 9 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 9 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 9 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 10 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 10 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 10 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 10 + b.eq .Ldec_finish_first_blocks // branch if AES-192 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #176] // load rk11, rk12 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 11 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 11 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 11 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 11 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 12 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 12 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 12 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 12 .Ldec_finish_first_blocks: - cmp $input_ptr, $main_end_input_ptr // check if we have <= 4 blocks - trn1 $acc_h.2d, $h3.2d, $h4.2d // h4h | h3h - trn2 $h34k.2d, $h3.2d, $h4.2d // h4l | h3l - trn1 $t0.2d, $h1.2d, $h2.2d // h2h | h1h - trn2 $h12k.2d, $h1.2d, $h2.2d // h2l | h1l - eor $h34k.16b, $h34k.16b, $acc_h.16b // h4k | h3k - aese $ctr1b, $rkNm1 // AES block 1 - round N-1 - aese $ctr2b, $rkNm1 // AES block 2 - round N-1 - eor $h12k.16b, $h12k.16b, $t0.16b // h2k | h1k - aese $ctr3b, $rkNm1 // AES block 3 - round N-1 - aese $ctr0b, $rkNm1 // AES block 0 - round N-1 - b.ge .Ldec_tail // handle tail - - ldr $res0q, [$input_ptr, #0] // AES block 0 - load ciphertext - ldr $res1q, [$input_ptr, #16] // AES block 1 - load ciphertext - rev $ctr32w, $rctr32w // CTR block 4 - eor $ctr0b, $res0b, $ctr0b // AES block 0 - result - eor $ctr1b, $res1b, $ctr1b // AES block 1 - result - rev64 $res1b, $res1b // GHASH block 1 - ldr $res3q, [$input_ptr, #48] // AES block 3 - load ciphertext - mov $output_h0, $ctr0.d[1] // AES block 0 - mov high - mov $output_l0, $ctr0.d[0] // AES block 0 - mov low - rev64 $res0b, $res0b // GHASH block 0 - add $rctr32w, $rctr32w, #1 // CTR block 4 - fmov $ctr0d, $ctr96_b64x // CTR block 4 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4 - fmov $ctr0.d[1], $ctr32x // CTR block 4 - rev $ctr32w, $rctr32w // CTR block 5 - add $rctr32w, $rctr32w, #1 // CTR block 5 - mov $output_l1, $ctr1.d[0] // AES block 1 - mov low - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 5 - mov $output_h1, $ctr1.d[1] // AES block 1 - mov high - eor $output_h0, $output_h0, $rkN_h // AES block 0 - round N high - eor $output_l0, $output_l0, $rkN_l // AES block 0 - round N low - stp $output_l0, $output_h0, [$output_ptr], #16 // AES block 0 - store result - fmov $ctr1d, $ctr96_b64x // CTR block 5 - ldr $res2q, [$input_ptr, #32] // AES block 2 - load ciphertext - add $input_ptr, $input_ptr, #64 // AES input_ptr update - fmov $ctr1.d[1], $ctr32x // CTR block 5 - rev $ctr32w, $rctr32w // CTR block 6 - add $rctr32w, $rctr32w, #1 // CTR block 6 - eor $output_l1, $output_l1, $rkN_l // AES block 1 - round N low - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 6 - eor $output_h1, $output_h1, $rkN_h // AES block 1 - round N high - stp $output_l1, $output_h1, [$output_ptr], #16 // AES block 1 - store result - eor $ctr2b, $res2b, $ctr2b // AES block 2 - result - cmp $input_ptr, $main_end_input_ptr // check if we have <= 8 blocks - b.ge .Ldec_prepretail // do prepretail - -.Ldec_main_loop: // main loop start - mov $output_l2, $ctr2.d[0] // AES block 4k+2 - mov low - ext $acc_lb, $acc_lb, $acc_lb, #8 // PRE 0 - eor $ctr3b, $res3b, $ctr3b // AES block 4k+3 - result - aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 0 - mov $output_h2, $ctr2.d[1] // AES block 4k+2 - mov high - aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 0 - fmov $ctr2d, $ctr96_b64x // CTR block 4k+6 - fmov $ctr2.d[1], $ctr32x // CTR block 4k+6 - eor $res0b, $res0b, $acc_lb // PRE 1 - rev $ctr32w, $rctr32w // CTR block 4k+7 - aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 1 - mov $output_h3, $ctr3.d[1] // AES block 4k+3 - mov high - aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 1 - mov $output_l3, $ctr3.d[0] // AES block 4k+3 - mov low - pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 4k - high - mov $t0d, $res0.d[1] // GHASH block 4k - mid - fmov $ctr3d, $ctr96_b64x // CTR block 4k+7 - aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 2 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+7 - aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 0 - fmov $ctr3.d[1], $ctr32x // CTR block 4k+7 - aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 2 - eor $t0.8b, $t0.8b, $res0.8b // GHASH block 4k - mid - aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 3 - eor $output_h2, $output_h2, $rkN_h // AES block 4k+2 - round N high - aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 1 - mov $acc_md, $h34k.d[1] // GHASH block 4k - mid - aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 3 - rev64 $res2b, $res2b // GHASH block 4k+2 - aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 0 - eor $output_l2, $output_l2, $rkN_l // AES block 4k+2 - round N low - aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 2 - stp $output_l2, $output_h2, [$output_ptr], #16 // AES block 4k+2 - store result - pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 4k - low - pmull2 $t1.1q, $res1.2d, $h3.2d // GHASH block 4k+1 - high - aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 3 - rev64 $res3b, $res3b // GHASH block 4k+3 - pmull $acc_m.1q, $t0.1d, $acc_m.1d // GHASH block 4k - mid - eor $output_l3, $output_l3, $rkN_l // AES block 4k+3 - round N low - pmull $t2.1q, $res1.1d, $h3.1d // GHASH block 4k+1 - low - eor $output_h3, $output_h3, $rkN_h // AES block 4k+3 - round N high - eor $acc_hb, $acc_hb, $t1.16b // GHASH block 4k+1 - high - aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 4 - aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 1 - mov $t3d, $res1.d[1] // GHASH block 4k+1 - mid - aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 4 - eor $acc_lb, $acc_lb, $t2.16b // GHASH block 4k+1 - low - aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 5 - add $rctr32w, $rctr32w, #1 // CTR block 4k+7 - aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 2 - mov $t6d, $res2.d[1] // GHASH block 4k+2 - mid - aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 4 - eor $t3.8b, $t3.8b, $res1.8b // GHASH block 4k+1 - mid - pmull $t5.1q, $res2.1d, $h2.1d // GHASH block 4k+2 - low - aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 3 - eor $t6.8b, $t6.8b, $res2.8b // GHASH block 4k+2 - mid - aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 5 - aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 5 - eor $acc_lb, $acc_lb, $t5.16b // GHASH block 4k+2 - low - pmull $t3.1q, $t3.1d, $h34k.1d // GHASH block 4k+1 - mid - rev $ctr32w, $rctr32w // CTR block 4k+8 - aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 6 - ins $t6.d[1], $t6.d[0] // GHASH block 4k+2 - mid - aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 6 - add $rctr32w, $rctr32w, #1 // CTR block 4k+8 - aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 4 - aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 7 - eor $acc_mb, $acc_mb, $t3.16b // GHASH block 4k+1 - mid - aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 7 - pmull2 $t4.1q, $res2.2d, $h2.2d // GHASH block 4k+2 - high - mov $t9d, $res3.d[1] // GHASH block 4k+3 - mid - aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 5 - pmull2 $t6.1q, $t6.2d, $h12k.2d // GHASH block 4k+2 - mid - aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 8 - eor $acc_hb, $acc_hb, $t4.16b // GHASH block 4k+2 - high - aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 6 - pmull $t8.1q, $res3.1d, $h1.1d // GHASH block 4k+3 - low - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+8 - eor $acc_mb, $acc_mb, $t6.16b // GHASH block 4k+2 - mid - pmull2 $t7.1q, $res3.2d, $h1.2d // GHASH block 4k+3 - high - cmp $rounds, #12 // setup flags for AES-128/192/256 check - eor $t9.8b, $t9.8b, $res3.8b // GHASH block 4k+3 - mid - aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 8 - aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 6 - eor $acc_hb, $acc_hb, $t7.16b // GHASH block 4k+3 - high - pmull $t9.1q, $t9.1d, $h12k.1d // GHASH block 4k+3 - mid - movi $mod_constant.8b, #0xc2 - aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 7 - eor $acc_lb, $acc_lb, $t8.16b // GHASH block 4k+3 - low - aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 7 - shl $mod_constantd, $mod_constantd, #56 // mod_constant - aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 8 - eor $acc_mb, $acc_mb, $t9.16b // GHASH block 4k+3 - mid - aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 8 - b.lt .Ldec_main_loop_continue // branch if AES-128 - - aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 9 - aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 9 - aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 9 - aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 9 - aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 10 - aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 10 - aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 10 - aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 10 - b.eq .Ldec_main_loop_continue // branch if AES-192 - - aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 11 - aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 11 - aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 11 - aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 11 - aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 12 - aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 12 - aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 12 - aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 12 - + ldr $rk9_11_tmpq, [$input_l1] // load rkN + cmp $input_ptr, $main_end_input_ptr // check if we have <= 4 blocks + eor $h34k.16b, $h34k.16b, $acc_h.16b // h4k | h3k + aese $ctr1.16b, $rkNm1.16b // AES block 1 - round N-1 + aese $ctr2.16b, $rkNm1.16b // AES block 2 - round N-1 + aese $ctr3.16b, $rkNm1.16b // AES block 3 - round N-1 + aese $ctr0.16b, $rkNm1.16b // AES block 0 - round N-1 + trn1 $t0.2d, $h1.2d, $h2.2d // h2h | h1h + eor $h12k.16b, $h12k.16b, $t0.16b // h2k | h1k + b.ge .Ldec_tail // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value ($v_rkN). This avoids several fmovs. + ldp $res2q, $res3q, [$input_ptr, #32] // AES blocks 2,3 load ciphertext + ldp $res0q, $res1q, [$input_ptr], #64 // AES blocks 0,1 load ciphertext + PLATFORM_EOR3($ctr0, $res0, $ctr0, $rk9_11_tmp) // AES block 0 - result + PLATFORM_EOR3($ctr1, $res1, $ctr1, $rk9_11_tmp) // AES block 1 - result + PLATFORM_EOR3($ctr2, $res2, $ctr2, $rk9_11_tmp) // AES block 2 - result + PLATFORM_EOR3($ctr3, $res3, $ctr3, $rk9_11_tmp) // AES block 3 - result + st1 { $ctr0.16b, $ctr1.16b, $ctr2.16b, $ctr3.16b}, [$output_ptr], #64 // AES blocks 0-3 - store result + ldr $ctr0q, [sp, #$ctr0_sp_offset] + ldr $ctr1q, [sp, #$ctr1_sp_offset] + ldr $ctr2q, [sp, #$ctr2_sp_offset] + ldr $ctr3q, [sp, #$ctr3_sp_offset] + rev $tmp_gpr_w, $ctr32w + str $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #1 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #2 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #3 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}] + add $ctr32w, $ctr32w, #4 + cmp $input_ptr, $main_end_input_ptr // check if we have <= 4 blocks + b.ge .Ldec_prepretail // do prepretail +.Ldec_main_loop: // main loop start + aese $ctr0.16b, $rk0.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 0 + aese $ctr1.16b, $rk0.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 0 + aese $ctr2.16b, $rk0.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 0 + aese $ctr3.16b, $rk0.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 0 + aese $ctr0.16b, $rk1.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 1 + aese $ctr1.16b, $rk1.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 1 + aese $ctr2.16b, $rk1.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 1 + aese $ctr3.16b, $rk1.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 1 + rev64 $res0.16b, $res0.16b // GHASH block 4k + rev64 $res1.16b, $res1.16b // GHASH block 4k+1 + rev64 $res2.16b, $res2.16b // GHASH block 4k+2 + rev64 $res3.16b, $res3.16b // GHASH block 4k+3 + aese $ctr0.16b, $rk2.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 2 + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 // PRE 0 + aese $ctr1.16b, $rk2.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 2 + eor $res0.16b, $res0.16b, $acc_l.16b // PRE 1 + aese $ctr2.16b, $rk2.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 2 + pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 4k - low + aese $ctr3.16b, $rk2.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 2 + pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 4k - high + aese $ctr0.16b, $rk3.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 3 + mov $acc_md, $h34k.d[1] // GHASH block 4k - mid + aese $ctr1.16b, $rk3.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 3 + mov $t0d, $res0.d[1] // GHASH block 4k - mid + aese $ctr2.16b, $rk3.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 3 + eor $t0.8b, $t0.8b, $res0.8b // GHASH block 4k - mid + aese $ctr3.16b, $rk3.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 3 + aese $ctr0.16b, $rk4.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 4 + pmull2 $t1.1q, $res1.2d, $h3.2d // GHASH block 4k+1 - high + aese $ctr1.16b, $rk4.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 4 + eor $acc_h.16b, $acc_h.16b, $t1.16b // GHASH block 4k+1 - high + aese $ctr2.16b, $rk4.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 4 + mov $t3d, $res1.d[1] // GHASH block 4k+1 - mid + aese $ctr3.16b, $rk4.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 4 + pmull $acc_m.1q, $t0.1d, $acc_m.1d // GHASH block 4k - mid + aese $ctr0.16b, $rk5.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 5 + pmull $t2.1q, $res1.1d, $h3.1d // GHASH block 4k+1 - low + aese $ctr1.16b, $rk5.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 5 + eor $t3.8b, $t3.8b, $res1.8b // GHASH block 4k+1 - mid + aese $ctr3.16b, $rk5.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 5 + aese $ctr2.16b, $rk5.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 5 + pmull $t5.1q, $res2.1d, $h2.1d // GHASH block 4k+2 - low + aese $ctr0.16b, $rk6.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 6 + PLATFORM_EOR3($acc_l, $acc_l, $t2, $t5) // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese $ctr1.16b, $rk6.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 6 + mov $t6d, $res2.d[1] // GHASH block 4k+2 - mid + aese $ctr2.16b, $rk6.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 6 + eor $t6.8b, $t6.8b, $res2.8b // GHASH block 4k+2 - mid + pmull $t3.1q, $t3.1d, $h34k.1d // GHASH block 4k+1 - mid + ins $t6.d[1], $t6.d[0] // GHASH block 4k+2 - mid + aese $ctr3.16b, $rk6.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 6 + pmull2 $rk10_12.1q, $res2.2d, $h2.2d // GHASH block 4k+2 - high + mov $t9d, $res3.d[1] // GHASH block 4k+3 - mid + aese $ctr0.16b, $rk7.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 7 + pmull2 $t6.1q, $t6.2d, $h12k.2d // GHASH block 4k+2 - mid + aese $ctr1.16b, $rk7.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 7 + pmull $rk9_11_tmp.1q, $res3.1d, $h1.1d // GHASH block 4k+3 - low + aese $ctr2.16b, $rk7.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 7 + PLATFORM_EOR3($acc_m, $acc_m, $t3, $t6) // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese $ctr3.16b, $rk7.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 7 + pmull2 $t7.1q, $res3.2d, $h1.2d // GHASH block 4k+3 - high + eor $t9.8b, $t9.8b, $res3.8b // GHASH block 4k+3 - mid + PLATFORM_EOR3($acc_h, $acc_h, $rk10_12, $t7) // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese $ctr0.16b, $rk8.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 8 + aese $ctr1.16b, $rk8.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 8 + aese $ctr2.16b, $rk8.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 8 + pmull $t9.1q, $t9.1d, $h12k.1d // GHASH block 4k+3 - mid + ldr $mod_constantd, [sp, #$mod_constant_sp_offset] + aese $ctr3.16b, $rk8.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 8 + pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid + PLATFORM_EOR3($acc_m, $acc_m, $t9, $mod_t) // GHASH block 4k+3 - mid & MODULO - fold into mid + eor $acc_l.16b, $acc_l.16b, $rk9_11_tmp.16b // GHASH block 4k+3 - low + eor $t9.16b, $acc_l.16b, $acc_h.16b // MODULO - karatsuba tidy up + ext $acc_h.16b, $acc_h.16b, $acc_h.16b, #8 // MODULO - other top alignment + PLATFORM_EOR3($acc_m, $acc_m, $t9, $acc_h) // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull $mod_constant.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low + ext $acc_m.16b, $acc_m.16b, $acc_m.16b, #8 // MODULO - other mid alignment + PLATFORM_EOR3($acc_l, $acc_l, $mod_constant, $acc_m) // MODULO - fold into low + cmp $rounds_w, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_main_loop_continue // branch if AES-128 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #144] // load rk9, rk10 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 9 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 9 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 9 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 9 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 10 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 10 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 10 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 10 + b.eq .Ldec_main_loop_continue // branch if AES-192 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #176] // load rk11, rk12 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 11 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 11 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 11 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 11 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 4k+4 - round 12 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 4k+5 - round 12 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 4k+6 - round 12 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 4k+7 - round 12 .Ldec_main_loop_continue: - pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid - eor $t9.16b, $acc_lb, $acc_hb // MODULO - karatsuba tidy up - ldr $res0q, [$input_ptr, #0] // AES block 4k+4 - load ciphertext - aese $ctr0b, $rkNm1 // AES block 4k+4 - round N-1 - ext $acc_hb, $acc_hb, $acc_hb, #8 // MODULO - other top alignment - eor $acc_mb, $acc_mb, $t9.16b // MODULO - karatsuba tidy up - ldr $res1q, [$input_ptr, #16] // AES block 4k+5 - load ciphertext - eor $ctr0b, $res0b, $ctr0b // AES block 4k+4 - result - stp $output_l3, $output_h3, [$output_ptr], #16 // AES block 4k+3 - store result - eor $acc_mb, $acc_mb, $mod_t.16b // MODULO - fold into mid - ldr $res3q, [$input_ptr, #48] // AES block 4k+7 - load ciphertext - ldr $res2q, [$input_ptr, #32] // AES block 4k+6 - load ciphertext - mov $output_h0, $ctr0.d[1] // AES block 4k+4 - mov high - eor $acc_mb, $acc_mb, $acc_hb // MODULO - fold into mid - aese $ctr1b, $rkNm1 // AES block 4k+5 - round N-1 - add $input_ptr, $input_ptr, #64 // AES input_ptr update - mov $output_l0, $ctr0.d[0] // AES block 4k+4 - mov low - fmov $ctr0d, $ctr96_b64x // CTR block 4k+8 - fmov $ctr0.d[1], $ctr32x // CTR block 4k+8 - pmull $mod_constant.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low - eor $ctr1b, $res1b, $ctr1b // AES block 4k+5 - result - rev $ctr32w, $rctr32w // CTR block 4k+9 - aese $ctr2b, $rkNm1 // AES block 4k+6 - round N-1 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+9 - cmp $input_ptr, $main_end_input_ptr // LOOP CONTROL - add $rctr32w, $rctr32w, #1 // CTR block 4k+9 - eor $output_l0, $output_l0, $rkN_l // AES block 4k+4 - round N low - eor $output_h0, $output_h0, $rkN_h // AES block 4k+4 - round N high - mov $output_h1, $ctr1.d[1] // AES block 4k+5 - mov high - eor $ctr2b, $res2b, $ctr2b // AES block 4k+6 - result - eor $acc_lb, $acc_lb, $mod_constant.16b // MODULO - fold into low - mov $output_l1, $ctr1.d[0] // AES block 4k+5 - mov low - fmov $ctr1d, $ctr96_b64x // CTR block 4k+9 - ext $acc_mb, $acc_mb, $acc_mb, #8 // MODULO - other mid alignment - fmov $ctr1.d[1], $ctr32x // CTR block 4k+9 - rev $ctr32w, $rctr32w // CTR block 4k+10 - add $rctr32w, $rctr32w, #1 // CTR block 4k+10 - aese $ctr3b, $rkNm1 // AES block 4k+7 - round N-1 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+10 - rev64 $res1b, $res1b // GHASH block 4k+5 - eor $output_h1, $output_h1, $rkN_h // AES block 4k+5 - round N high - stp $output_l0, $output_h0, [$output_ptr], #16 // AES block 4k+4 - store result - eor $output_l1, $output_l1, $rkN_l // AES block 4k+5 - round N low - stp $output_l1, $output_h1, [$output_ptr], #16 // AES block 4k+5 - store result - rev64 $res0b, $res0b // GHASH block 4k+4 - eor $acc_lb, $acc_lb, $acc_mb // MODULO - fold into low - b.lt .Ldec_main_loop - -.Ldec_prepretail: // PREPRETAIL - ext $acc_lb, $acc_lb, $acc_lb, #8 // PRE 0 - mov $output_l2, $ctr2.d[0] // AES block 4k+2 - mov low - eor $ctr3b, $res3b, $ctr3b // AES block 4k+3 - result - aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 0 - mov $output_h2, $ctr2.d[1] // AES block 4k+2 - mov high - aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 0 - fmov $ctr2d, $ctr96_b64x // CTR block 4k+6 - fmov $ctr2.d[1], $ctr32x // CTR block 4k+6 - rev $ctr32w, $rctr32w // CTR block 4k+7 - eor $res0b, $res0b, $acc_lb // PRE 1 - rev64 $res2b, $res2b // GHASH block 4k+2 - orr $ctr32x, $ctr96_t32x, $ctr32x, lsl #32 // CTR block 4k+7 - mov $output_l3, $ctr3.d[0] // AES block 4k+3 - mov low - aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 1 - mov $output_h3, $ctr3.d[1] // AES block 4k+3 - mov high - pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 4k - low - mov $t0d, $res0.d[1] // GHASH block 4k - mid - fmov $ctr3d, $ctr96_b64x // CTR block 4k+7 - pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 4k - high - fmov $ctr3.d[1], $ctr32x // CTR block 4k+7 - aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 0 - mov $acc_md, $h34k.d[1] // GHASH block 4k - mid - aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 1 - eor $t0.8b, $t0.8b, $res0.8b // GHASH block 4k - mid - pmull2 $t1.1q, $res1.2d, $h3.2d // GHASH block 4k+1 - high - aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 1 - rev64 $res3b, $res3b // GHASH block 4k+3 - aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 0 - pmull $acc_m.1q, $t0.1d, $acc_m.1d // GHASH block 4k - mid - eor $acc_hb, $acc_hb, $t1.16b // GHASH block 4k+1 - high - pmull $t2.1q, $res1.1d, $h3.1d // GHASH block 4k+1 - low - aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 1 - mov $t3d, $res1.d[1] // GHASH block 4k+1 - mid - aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 2 - aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 2 - eor $acc_lb, $acc_lb, $t2.16b // GHASH block 4k+1 - low - aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 2 - aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 3 - mov $t6d, $res2.d[1] // GHASH block 4k+2 - mid - aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 2 - eor $t3.8b, $t3.8b, $res1.8b // GHASH block 4k+1 - mid - pmull $t5.1q, $res2.1d, $h2.1d // GHASH block 4k+2 - low - aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 4 - aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 3 - eor $t6.8b, $t6.8b, $res2.8b // GHASH block 4k+2 - mid - pmull $t3.1q, $t3.1d, $h34k.1d // GHASH block 4k+1 - mid - aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 5 - eor $acc_lb, $acc_lb, $t5.16b // GHASH block 4k+2 - low - aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 4 - pmull2 $t7.1q, $res3.2d, $h1.2d // GHASH block 4k+3 - high - eor $acc_mb, $acc_mb, $t3.16b // GHASH block 4k+1 - mid - pmull2 $t4.1q, $res2.2d, $h2.2d // GHASH block 4k+2 - high - aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 5 - ins $t6.d[1], $t6.d[0] // GHASH block 4k+2 - mid - aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 3 - aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 3 - eor $acc_hb, $acc_hb, $t4.16b // GHASH block 4k+2 - high - pmull $t8.1q, $res3.1d, $h1.1d // GHASH block 4k+3 - low - aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 4 - mov $t9d, $res3.d[1] // GHASH block 4k+3 - mid - aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 4 - pmull2 $t6.1q, $t6.2d, $h12k.2d // GHASH block 4k+2 - mid - aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 5 - eor $t9.8b, $t9.8b, $res3.8b // GHASH block 4k+3 - mid - aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 5 - aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 6 - eor $acc_mb, $acc_mb, $t6.16b // GHASH block 4k+2 - mid - aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 6 - aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 6 - movi $mod_constant.8b, #0xc2 - aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 6 - eor $acc_lb, $acc_lb, $t8.16b // GHASH block 4k+3 - low - pmull $t9.1q, $t9.1d, $h12k.1d // GHASH block 4k+3 - mid - aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 7 - cmp $rounds, #12 // setup flags for AES-128/192/256 check - eor $acc_hb, $acc_hb, $t7.16b // GHASH block 4k+3 - high - aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 7 - aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 7 - eor $acc_mb, $acc_mb, $t9.16b // GHASH block 4k+3 - mid - aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 8 - aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 7 - eor $t9.16b, $acc_lb, $acc_hb // MODULO - karatsuba tidy up - aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 8 - aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 8 - shl $mod_constantd, $mod_constantd, #56 // mod_constant - aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 8 - b.lt .Ldec_finish_prepretail // branch if AES-128 - - aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 9 - aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 9 - aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 9 - aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 9 - aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 10 - aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 10 - aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 10 - aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 10 - b.eq .Ldec_finish_prepretail // branch if AES-192 - - aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 11 - aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 11 - aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 11 - aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b // AES block 4k+6 - round 12 - aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 11 - aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b // AES block 4k+5 - round 12 - aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b // AES block 4k+4 - round 12 - aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b // AES block 4k+7 - round 12 - + ldr $rk9_11_tmpq, [$input_l1] // load rkN + ldp $res2q, $res3q, [$input_ptr, #32] // AES blocks 2,3 load ciphertext + ldp $res0q, $res1q, [$input_ptr], #64 // AES blocks 0,1 load ciphertext + aese $ctr0.16b, $rkNm1.16b // AES block 4k+4 - round N-1 + PLATFORM_EOR3($ctr0, $res0, $ctr0, $rk9_11_tmp) // AES block 4k+4 - result + aese $ctr1.16b, $rkNm1.16b // AES block 4k+5 - round N-1 + PLATFORM_EOR3($ctr1, $res1, $ctr1, $rk9_11_tmp) // AES block 4k+5 - result + aese $ctr2.16b, $rkNm1.16b // AES block 4k+6 - round N-1 + PLATFORM_EOR3($ctr2, $res2, $ctr2, $rk9_11_tmp) // AES block 4k+6 - result + aese $ctr3.16b, $rkNm1.16b // AES block 4k+7 - round N-1 + PLATFORM_EOR3($ctr3, $res3, $ctr3, $rk9_11_tmp) // AES block 4k+7 - result + st1 { $ctr0.16b, $ctr1.16b, $ctr2.16b, $ctr3.16b}, [$output_ptr], #64 // AES blocks 4k+4-7 - store result + ldr $ctr0q, [sp, #$ctr0_sp_offset] + ldr $ctr1q, [sp, #$ctr1_sp_offset] + ldr $ctr2q, [sp, #$ctr2_sp_offset] + ldr $ctr3q, [sp, #$ctr3_sp_offset] + rev $tmp_gpr_w, $ctr32w + str $tmp_gpr_w, [sp, #@{[ $ctr0_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #1 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr1_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #2 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr2_sp_offset + 12 ]}] + add $tmp_gpr_w, $ctr32w, #3 + rev $tmp_gpr_w, $tmp_gpr_w + str $tmp_gpr_w, [sp, #@{[ $ctr3_sp_offset + 12 ]}] + add $ctr32w, $ctr32w, #4 + cmp $input_ptr, $main_end_input_ptr + b.lt .Ldec_main_loop +.Ldec_prepretail: // PREPRETAIL + rev64 $res0.16b, $res0.16b // GHASH block 0 + rev64 $res1.16b, $res1.16b // GHASH block 1 + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 // PRE 0 + aese $ctr0.16b, $rk0.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 0 + aese $ctr1.16b, $rk0.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 0 + eor $res0.16b, $res0.16b, $acc_l.16b // PRE 1 + rev64 $res2.16b, $res2.16b // GHASH block 2 + aese $ctr1.16b, $rk1.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 1 + pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH block 0 - low + mov $t0d, $res0.d[1] // GHASH block 0 - mid + pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH block 0 - high + aese $ctr2.16b, $rk0.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 0 + mov $acc_md, $h34k.d[1] // GHASH block 0 - mid + aese $ctr0.16b, $rk1.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 1 + eor $t0.8b, $t0.8b, $res0.8b // GHASH block 0 - mid + pmull2 $t1.1q, $res1.2d, $h3.2d // GHASH block 1 - high + aese $ctr2.16b, $rk1.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 1 + rev64 $res3.16b, $res3.16b // GHASH block 3 + aese $ctr3.16b, $rk0.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 0 + pmull $acc_m.1q, $t0.1d, $acc_m.1d // GHASH block 0 - mid + eor $acc_h.16b, $acc_h.16b, $t1.16b // GHASH block 1 - high + pmull $t2.1q, $res1.1d, $h3.1d // GHASH block 1 - low + aese $ctr3.16b, $rk1.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 1 + mov $t3d, $res1.d[1] // GHASH block 1 - mid + aese $ctr0.16b, $rk2.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 2 + aese $ctr1.16b, $rk2.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 2 + eor $acc_l.16b, $acc_l.16b, $t2.16b // GHASH block 1 - low + aese $ctr2.16b, $rk2.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 2 + aese $ctr0.16b, $rk3.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 3 + mov $t6d, $res2.d[1] // GHASH block 2 - mid + aese $ctr3.16b, $rk2.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 2 + eor $t3.8b, $t3.8b, $res1.8b // GHASH block 1 - mid + pmull $t5.1q, $res2.1d, $h2.1d // GHASH block 2 - low + aese $ctr0.16b, $rk4.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 4 + aese $ctr3.16b, $rk3.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 3 + eor $t6.8b, $t6.8b, $res2.8b // GHASH block 2 - mid + pmull $t3.1q, $t3.1d, $h34k.1d // GHASH block 1 - mid + aese $ctr0.16b, $rk5.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 5 + eor $acc_l.16b, $acc_l.16b, $t5.16b // GHASH block 2 - low + aese $ctr3.16b, $rk4.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 4 + pmull2 $t7.1q, $res3.2d, $h1.2d // GHASH block 3 - high + eor $acc_m.16b, $acc_m.16b, $t3.16b // GHASH block 1 - mid + pmull2 $t4.1q, $res2.2d, $h2.2d // GHASH block 2 - high + aese $ctr3.16b, $rk5.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 5 + ins $t6.d[1], $t6.d[0] // GHASH block 2 - mid + aese $ctr2.16b, $rk3.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 3 + aese $ctr1.16b, $rk3.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 3 + PLATFORM_EOR3($acc_h, $acc_h, $t4, $t7) // GHASH block 2 - high & GHASH block 3 - high + pmull $t8.1q, $res3.1d, $h1.1d // GHASH block 3 - low + aese $ctr2.16b, $rk4.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 4 + mov $t9d, $res3.d[1] // GHASH block 3 - mid + aese $ctr1.16b, $rk4.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 4 + pmull2 $t6.1q, $t6.2d, $h12k.2d // GHASH block 2 - mid + aese $ctr2.16b, $rk5.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 5 + eor $t9.8b, $t9.8b, $res3.8b // GHASH block 3 - mid + aese $ctr1.16b, $rk5.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 5 + aese $ctr3.16b, $rk6.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 6 + eor $acc_m.16b, $acc_m.16b, $t6.16b // GHASH block 2 - mid + aese $ctr2.16b, $rk6.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 6 + aese $ctr0.16b, $rk6.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 6 + movi $mod_constant.8b, #0xc2 + aese $ctr1.16b, $rk6.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 6 + eor $acc_l.16b, $acc_l.16b, $t8.16b // GHASH block 3 - low + pmull $t9.1q, $t9.1d, $h12k.1d // GHASH block 3 - mid + aese $ctr3.16b, $rk7.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 7 + aese $ctr1.16b, $rk7.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 7 + aese $ctr0.16b, $rk7.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 7 + eor $acc_m.16b, $acc_m.16b, $t9.16b // GHASH block 3 - mid + aese $ctr3.16b, $rk8.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 8 + aese $ctr2.16b, $rk7.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 7 + eor $t9.16b, $acc_l.16b, $acc_h.16b // MODULO - karatsuba tidy up + aese $ctr1.16b, $rk8.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 8 + aese $ctr0.16b, $rk8.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 8 + shl $mod_constantd, $mod_constantd, #56 // mod_constant + aese $ctr2.16b, $rk8.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 8 + cmp $rounds_w, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_finish_prepretail // branch if AES-128 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #144] // load rk9, rk10 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 9 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 9 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 9 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 9 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 10 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 10 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 10 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 10 + b.eq .Ldec_finish_prepretail // branch if AES-192 + ldp $rk9_11_tmpq, $rk10_12q, [$cc, #176] // load rk11, rk12 + aese $ctr0.16b, $rk9_11_tmp.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 11 + aese $ctr1.16b, $rk9_11_tmp.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 11 + aese $ctr2.16b, $rk9_11_tmp.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 11 + aese $ctr3.16b, $rk9_11_tmp.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 11 + aese $ctr0.16b, $rk10_12.16b \n aesmc $ctr0.16b, $ctr0.16b // AES block 0 - round 12 + aese $ctr1.16b, $rk10_12.16b \n aesmc $ctr1.16b, $ctr1.16b // AES block 1 - round 12 + aese $ctr2.16b, $rk10_12.16b \n aesmc $ctr2.16b, $ctr2.16b // AES block 2 - round 12 + aese $ctr3.16b, $rk10_12.16b \n aesmc $ctr3.16b, $ctr3.16b // AES block 3 - round 12 .Ldec_finish_prepretail: - eor $acc_mb, $acc_mb, $t9.16b // MODULO - karatsuba tidy up - pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid - ext $acc_hb, $acc_hb, $acc_hb, #8 // MODULO - other top alignment - eor $acc_mb, $acc_mb, $mod_t.16b // MODULO - fold into mid - eor $output_h2, $output_h2, $rkN_h // AES block 4k+2 - round N high - eor $output_l3, $output_l3, $rkN_l // AES block 4k+3 - round N low - eor $acc_mb, $acc_mb, $acc_hb // MODULO - fold into mid - add $rctr32w, $rctr32w, #1 // CTR block 4k+7 - eor $output_l2, $output_l2, $rkN_l // AES block 4k+2 - round N low - pmull $mod_constant.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low - eor $output_h3, $output_h3, $rkN_h // AES block 4k+3 - round N high - stp $output_l2, $output_h2, [$output_ptr], #16 // AES block 4k+2 - store result - ext $acc_mb, $acc_mb, $acc_mb, #8 // MODULO - other mid alignment - stp $output_l3, $output_h3, [$output_ptr], #16 // AES block 4k+3 - store result - - eor $acc_lb, $acc_lb, $mod_constant.16b // MODULO - fold into low - aese $ctr1b, $rkNm1 // AES block 4k+5 - round N-1 - aese $ctr0b, $rkNm1 // AES block 4k+4 - round N-1 - aese $ctr3b, $rkNm1 // AES block 4k+7 - round N-1 - aese $ctr2b, $rkNm1 // AES block 4k+6 - round N-1 - eor $acc_lb, $acc_lb, $acc_mb // MODULO - fold into low - -.Ldec_tail: // TAIL - sub $main_end_input_ptr, $end_input_ptr, $input_ptr // main_end_input_ptr is number of bytes left to process - ld1 { $res1b}, [$input_ptr], #16 // AES block 4k+4 - load ciphertext - eor $ctr0b, $res1b, $ctr0b // AES block 4k+4 - result - mov $output_l0, $ctr0.d[0] // AES block 4k+4 - mov low - mov $output_h0, $ctr0.d[1] // AES block 4k+4 - mov high - ext $t0.16b, $acc_lb, $acc_lb, #8 // prepare final partial tag - cmp $main_end_input_ptr, #48 - eor $output_l0, $output_l0, $rkN_l // AES block 4k+4 - round N low - eor $output_h0, $output_h0, $rkN_h // AES block 4k+4 - round N high - b.gt .Ldec_blocks_more_than_3 - sub $rctr32w, $rctr32w, #1 - mov $ctr3b, $ctr2b - movi $acc_m.8b, #0 - movi $acc_l.8b, #0 - cmp $main_end_input_ptr, #32 - movi $acc_h.8b, #0 - mov $ctr2b, $ctr1b - b.gt .Ldec_blocks_more_than_2 - sub $rctr32w, $rctr32w, #1 - mov $ctr3b, $ctr1b - cmp $main_end_input_ptr, #16 - b.gt .Ldec_blocks_more_than_1 - sub $rctr32w, $rctr32w, #1 - b .Ldec_blocks_less_than_1 -.Ldec_blocks_more_than_3: // blocks left > 3 - rev64 $res0b, $res1b // GHASH final-3 block - ld1 { $res1b}, [$input_ptr], #16 // AES final-2 block - load ciphertext - stp $output_l0, $output_h0, [$output_ptr], #16 // AES final-3 block - store result - mov $acc_md, $h34k.d[1] // GHASH final-3 block - mid - eor $res0b, $res0b, $t0.16b // feed in partial tag - eor $ctr0b, $res1b, $ctr1b // AES final-2 block - result - mov $rk4d, $res0.d[1] // GHASH final-3 block - mid - mov $output_l0, $ctr0.d[0] // AES final-2 block - mov low - mov $output_h0, $ctr0.d[1] // AES final-2 block - mov high - eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-3 block - mid - movi $t0.8b, #0 // suppress further partial tag feed in - pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH final-3 block - high - pmull $acc_m.1q, $rk4v.1d, $acc_m.1d // GHASH final-3 block - mid - eor $output_l0, $output_l0, $rkN_l // AES final-2 block - round N low - pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH final-3 block - low - eor $output_h0, $output_h0, $rkN_h // AES final-2 block - round N high -.Ldec_blocks_more_than_2: // blocks left > 2 - rev64 $res0b, $res1b // GHASH final-2 block - ld1 { $res1b}, [$input_ptr], #16 // AES final-1 block - load ciphertext - eor $res0b, $res0b, $t0.16b // feed in partial tag - stp $output_l0, $output_h0, [$output_ptr], #16 // AES final-2 block - store result - eor $ctr0b, $res1b, $ctr2b // AES final-1 block - result - mov $rk4d, $res0.d[1] // GHASH final-2 block - mid - pmull $rk3q1, $res0.1d, $h3.1d // GHASH final-2 block - low - pmull2 $rk2q1, $res0.2d, $h3.2d // GHASH final-2 block - high - eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-2 block - mid - mov $output_l0, $ctr0.d[0] // AES final-1 block - mov low - mov $output_h0, $ctr0.d[1] // AES final-1 block - mov high - eor $acc_lb, $acc_lb, $rk3 // GHASH final-2 block - low - movi $t0.8b, #0 // suppress further partial tag feed in - pmull $rk4v.1q, $rk4v.1d, $h34k.1d // GHASH final-2 block - mid - eor $acc_hb, $acc_hb, $rk2 // GHASH final-2 block - high - eor $output_l0, $output_l0, $rkN_l // AES final-1 block - round N low - eor $acc_mb, $acc_mb, $rk4v.16b // GHASH final-2 block - mid - eor $output_h0, $output_h0, $rkN_h // AES final-1 block - round N high -.Ldec_blocks_more_than_1: // blocks left > 1 - stp $output_l0, $output_h0, [$output_ptr], #16 // AES final-1 block - store result - rev64 $res0b, $res1b // GHASH final-1 block - ld1 { $res1b}, [$input_ptr], #16 // AES final block - load ciphertext - eor $res0b, $res0b, $t0.16b // feed in partial tag - movi $t0.8b, #0 // suppress further partial tag feed in - mov $rk4d, $res0.d[1] // GHASH final-1 block - mid - eor $ctr0b, $res1b, $ctr3b // AES final block - result - pmull2 $rk2q1, $res0.2d, $h2.2d // GHASH final-1 block - high - eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-1 block - mid - pmull $rk3q1, $res0.1d, $h2.1d // GHASH final-1 block - low - mov $output_l0, $ctr0.d[0] // AES final block - mov low - ins $rk4v.d[1], $rk4v.d[0] // GHASH final-1 block - mid - mov $output_h0, $ctr0.d[1] // AES final block - mov high - pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d // GHASH final-1 block - mid - eor $output_l0, $output_l0, $rkN_l // AES final block - round N low - eor $acc_lb, $acc_lb, $rk3 // GHASH final-1 block - low - eor $acc_hb, $acc_hb, $rk2 // GHASH final-1 block - high - eor $acc_mb, $acc_mb, $rk4v.16b // GHASH final-1 block - mid - eor $output_h0, $output_h0, $rkN_h // AES final block - round N high -.Ldec_blocks_less_than_1: // blocks left <= 1 - and $bit_length, $bit_length, #127 // bit_length %= 128 - mvn $rkN_h, xzr // rkN_h = 0xffffffffffffffff - sub $bit_length, $bit_length, #128 // bit_length -= 128 - mvn $rkN_l, xzr // rkN_l = 0xffffffffffffffff - ldp $end_input_ptr, $main_end_input_ptr, [$output_ptr] // load existing bytes we need to not overwrite - neg $bit_length, $bit_length // bit_length = 128 - #bits in input (in range [1,128]) - and $bit_length, $bit_length, #127 // bit_length %= 128 - lsr $rkN_h, $rkN_h, $bit_length // rkN_h is mask for top 64b of last block - cmp $bit_length, #64 - csel $ctr32x, $rkN_l, $rkN_h, lt - csel $ctr96_b64x, $rkN_h, xzr, lt - fmov $ctr0d, $ctr32x // ctr0b is mask for last block - and $output_l0, $output_l0, $ctr32x - mov $ctr0.d[1], $ctr96_b64x - bic $end_input_ptr, $end_input_ptr, $ctr32x // mask out low existing bytes - rev $ctr32w, $rctr32w - bic $main_end_input_ptr, $main_end_input_ptr, $ctr96_b64x // mask out high existing bytes - orr $output_l0, $output_l0, $end_input_ptr - and $output_h0, $output_h0, $ctr96_b64x - orr $output_h0, $output_h0, $main_end_input_ptr - and $res1b, $res1b, $ctr0b // possibly partial last block has zeroes in highest bits - rev64 $res0b, $res1b // GHASH final block - eor $res0b, $res0b, $t0.16b // feed in partial tag - pmull $rk3q1, $res0.1d, $h1.1d // GHASH final block - low - mov $t0d, $res0.d[1] // GHASH final block - mid - eor $t0.8b, $t0.8b, $res0.8b // GHASH final block - mid - pmull2 $rk2q1, $res0.2d, $h1.2d // GHASH final block - high - pmull $t0.1q, $t0.1d, $h12k.1d // GHASH final block - mid - eor $acc_hb, $acc_hb, $rk2 // GHASH final block - high - eor $acc_lb, $acc_lb, $rk3 // GHASH final block - low - eor $acc_mb, $acc_mb, $t0.16b // GHASH final block - mid - movi $mod_constant.8b, #0xc2 - eor $t9.16b, $acc_lb, $acc_hb // MODULO - karatsuba tidy up - shl $mod_constantd, $mod_constantd, #56 // mod_constant - eor $acc_mb, $acc_mb, $t9.16b // MODULO - karatsuba tidy up - pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid - ext $acc_hb, $acc_hb, $acc_hb, #8 // MODULO - other top alignment - eor $acc_mb, $acc_mb, $mod_t.16b // MODULO - fold into mid - eor $acc_mb, $acc_mb, $acc_hb // MODULO - fold into mid - pmull $mod_constant.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low - ext $acc_mb, $acc_mb, $acc_mb, #8 // MODULO - other mid alignment - eor $acc_lb, $acc_lb, $mod_constant.16b // MODULO - fold into low - stp $output_l0, $output_h0, [$output_ptr] - str $ctr32w, [$counter, #12] // store the updated counter - eor $acc_lb, $acc_lb, $acc_mb // MODULO - fold into low - ext $acc_lb, $acc_lb, $acc_lb, #8 - rev64 $acc_lb, $acc_lb - mov x0, $len - st1 { $acc_l.16b }, [$current_tag] - ldp x19, x20, [sp, #16] - ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] - ldp d8, d9, [sp, #64] - ldp d10, d11, [sp, #80] - ldp d12, d13, [sp, #96] - ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 - AARCH64_VALIDATE_LINK_REGISTER - ret + eor $acc_m.16b, $acc_m.16b, $t9.16b // MODULO - karatsuba tidy up + pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid + ext $acc_h.16b, $acc_h.16b, $acc_h.16b, #8 // MODULO - other top alignment + PLATFORM_EOR3($acc_m, $acc_m, $mod_t, $acc_h) // MODULO - fold into mid + pmull $mod_constant.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low + ext $acc_m.16b, $acc_m.16b, $acc_m.16b, #8 // MODULO - other mid alignment + PLATFORM_EOR3($acc_l, $acc_l, $mod_constant, $acc_m) // MODULO - fold into low + aese $ctr1.16b, $rkNm1.16b // AES block 1 - round N-1 + aese $ctr0.16b, $rkNm1.16b // AES block 0 - round N-1 + aese $ctr3.16b, $rkNm1.16b // AES block 3 - round N-1 + aese $ctr2.16b, $rkNm1.16b // AES block 2 - round N-1 +.Ldec_tail: // TAIL + sub $main_end_input_ptr, $end_input_ptr, $input_ptr // main_end_input_ptr is number of bytes left to process + ld1 { $res1.16b}, [$input_ptr], #16 // AES block 0 - load ciphertext + eor $ctr0.16b, $res1.16b, $ctr0.16b // AES block 0 - result + mov $output_l0, $ctr0.d[0] // AES block 0 - mov low + mov $output_h0, $ctr0.d[1] // AES block 0 - mov high + ext $t0.16b, $acc_l.16b, $acc_l.16b, #8 // prepare final partial tag + eor $output_l0, $output_l0, $rkN_l // AES block 0 - round N low + eor $output_h0, $output_h0, $rkN_h // AES block 0 - round N high + cmp $main_end_input_ptr, #48 + b.gt .Ldec_blocks_more_than_3 + mov $ctr3.16b, $ctr2.16b + movi $acc_m.8b, #0 + movi $acc_l.8b, #0 + movi $acc_h.8b, #0 + mov $ctr2.16b, $ctr1.16b + cmp $main_end_input_ptr, #32 + b.gt .Ldec_blocks_more_than_2 + mov $ctr3.16b, $ctr1.16b + cmp $main_end_input_ptr, #16 + b.gt .Ldec_blocks_more_than_1 + b .Ldec_blocks_less_than_1 +.Ldec_blocks_more_than_3: // blocks left > 3 + rev64 $res0.16b, $res1.16b // GHASH final-3 block + ld1 { $res1.16b}, [$input_ptr], #16 // AES final-2 block - load ciphertext + stp $output_l0, $output_h0, [$output_ptr], #16 // AES final-3 block - store result + mov $acc_md, $h34k.d[1] // GHASH final-3 block - mid + eor $res0.16b, $res0.16b, $t0.16b // feed in partial tag + eor $ctr0.16b, $res1.16b, $ctr1.16b // AES final-2 block - result + mov $rk4d, $res0.d[1] // GHASH final-3 block - mid + mov $output_l0, $ctr0.d[0] // AES final-2 block - mov low + mov $output_h0, $ctr0.d[1] // AES final-2 block - mov high + eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-3 block - mid + movi $t0.8b, #0 // suppress further partial tag feed in + pmull2 $acc_h.1q, $res0.2d, $h4.2d // GHASH final-3 block - high + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d // GHASH final-3 block - mid + eor $output_l0, $output_l0, $rkN_l // AES final-2 block - round N low + pmull $acc_l.1q, $res0.1d, $h4.1d // GHASH final-3 block - low + eor $output_h0, $output_h0, $rkN_h // AES final-2 block - round N high +.Ldec_blocks_more_than_2: // blocks left > 2 + rev64 $res0.16b, $res1.16b // GHASH final-2 block + ld1 { $res1.16b}, [$input_ptr], #16 // AES final-1 block - load ciphertext + eor $res0.16b, $res0.16b, $t0.16b // feed in partial tag + stp $output_l0, $output_h0, [$output_ptr], #16 // AES final-2 block - store result + eor $ctr0.16b, $res1.16b, $ctr2.16b // AES final-1 block - result + mov $rk4d, $res0.d[1] // GHASH final-2 block - mid + pmull $rk3q1, $res0.1d, $h3.1d // GHASH final-2 block - low + pmull2 $rk2.1q, $res0.2d, $h3.2d // GHASH final-2 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-2 block - mid + mov $output_l0, $ctr0.d[0] // AES final-1 block - mov low + mov $output_h0, $ctr0.d[1] // AES final-1 block - mov high + eor $acc_l.16b, $acc_l.16b, $rk3.16b // GHASH final-2 block - low + movi $t0.8b, #0 // suppress further partial tag feed in + pmull $rk4v.1q, $rk4v.1d, $h34k.1d // GHASH final-2 block - mid + eor $acc_h.16b, $acc_h.16b, $rk2.16b // GHASH final-2 block - high + eor $output_l0, $output_l0, $rkN_l // AES final-1 block - round N low + eor $acc_m.16b, $acc_m.16b, $rk4v.16b // GHASH final-2 block - mid + eor $output_h0, $output_h0, $rkN_h // AES final-1 block - round N high +.Ldec_blocks_more_than_1: // blocks left > 1 + stp $output_l0, $output_h0, [$output_ptr], #16 // AES final-1 block - store result + rev64 $res0.16b, $res1.16b // GHASH final-1 block + ld1 { $res1.16b}, [$input_ptr], #16 // AES final block - load ciphertext + eor $res0.16b, $res0.16b, $t0.16b // feed in partial tag + movi $t0.8b, #0 // suppress further partial tag feed in + mov $rk4d, $res0.d[1] // GHASH final-1 block - mid + eor $ctr0.16b, $res1.16b, $ctr3.16b // AES final block - result + pmull2 $rk2q1, $res0.2d, $h2.2d // GHASH final-1 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b // GHASH final-1 block - mid + pmull $rk3q1, $res0.1d, $h2.1d // GHASH final-1 block - low + mov $output_l0, $ctr0.d[0] // AES final block - mov low + ins $rk4v.d[1], $rk4v.d[0] // GHASH final-1 block - mid + mov $output_h0, $ctr0.d[1] // AES final block - mov high + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d // GHASH final-1 block - mid + eor $output_l0, $output_l0, $rkN_l // AES final block - round N low + eor $acc_l.16b, $acc_l.16b, $rk3.16b // GHASH final-1 block - low + eor $acc_h.16b, $acc_h.16b, $rk2.16b // GHASH final-1 block - high + eor $acc_m.16b, $acc_m.16b, $rk4v.16b // GHASH final-1 block - mid + eor $output_h0, $output_h0, $rkN_h // AES final block - round N high +.Ldec_blocks_less_than_1: // blocks left <= 1 + add $T32, $T32, $bit_length, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev $W32, $W32 + str $W32, [$counter, #12] // store the updated counter + and $bit_length, $bit_length, #127 // bit_length %= 128 + mvn $rkN_h, xzr // rkN_h = 0xffffffffffffffff + sub $bit_length, $bit_length, #128 // bit_length -= 128 + mvn $rkN_l, xzr // rkN_l = 0xffffffffffffffff + ldp $end_input_ptr, $main_end_input_ptr, [$output_ptr] // load existing bytes we need to not overwrite + neg $bit_length, $bit_length // bit_length = 128 - #bits in input (in range [1,128]) + and $bit_length, $bit_length, #127 // bit_length %= 128 + lsr $rkN_h, $rkN_h, $bit_length // rkN_h is mask for top 64b of last block + cmp $bit_length, #64 + csel $ctr32x, $rkN_l, $rkN_h, lt + csel $ctr96_b64x, $rkN_h, xzr, lt + fmov $ctr0d, $ctr32x // ctr0b is mask for last block + and $output_l0, $output_l0, $ctr32x + mov $ctr0.d[1], $ctr96_b64x + bic $end_input_ptr, $end_input_ptr, $ctr32x // mask out low existing bytes + bic $main_end_input_ptr, $main_end_input_ptr, $ctr96_b64x // mask out high existing bytes + orr $output_l0, $output_l0, $end_input_ptr + and $output_h0, $output_h0, $ctr96_b64x + orr $output_h0, $output_h0, $main_end_input_ptr + and $res1.16b, $res1.16b, $ctr0.16b // possibly partial last block has zeroes in highest bits + rev64 $res0.16b, $res1.16b // GHASH final block + eor $res0.16b, $res0.16b, $t0.16b // feed in partial tag + pmull $rk3q1, $res0.1d, $h1.1d // GHASH final block - low + mov $t0d, $res0.d[1] // GHASH final block - mid + eor $t0.8b, $t0.8b, $res0.8b // GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d // GHASH final block - high + pmull $t0.1q, $t0.1d, $h12k.1d // GHASH final block - mid + eor $acc_h.16b, $acc_h.16b, $rk2.16b // GHASH final block - high + eor $acc_l.16b, $acc_l.16b, $rk3.16b // GHASH final block - low + eor $acc_m.16b, $acc_m.16b, $t0.16b // GHASH final block - mid + ldr $mod_constantd, [sp, #$mod_constant_sp_offset] + eor $t9.16b, $acc_l.16b, $acc_h.16b // MODULO - karatsuba tidy up + eor $acc_m.16b, $acc_m.16b, $t9.16b // MODULO - karatsuba tidy up + pmull $mod_t.1q, $acc_h.1d, $mod_constant.1d // MODULO - top 64b align with mid + ext $acc_h.16b, $acc_h.16b, $acc_h.16b, #8 // MODULO - other top alignment + eor $acc_m.16b, $acc_m.16b, $mod_t.16b // MODULO - fold into mid + eor $acc_m.16b, $acc_m.16b, $acc_h.16b // MODULO - fold into mid + pmull $mod_constant.1q, $acc_m.1d, $mod_constant.1d // MODULO - mid 64b align with low + ext $acc_m.16b, $acc_m.16b, $acc_m.16b, #8 // MODULO - other mid alignment + eor $acc_l.16b, $acc_l.16b, $mod_constant.16b // MODULO - fold into low + stp $output_l0, $output_h0, [$output_ptr] + eor $acc_l.16b, $acc_l.16b, $acc_m.16b // MODULO - fold into low + ext $acc_l.16b, $acc_l.16b, $acc_l.16b, #8 + rev64 $acc_l.16b, $acc_l.16b // Final Tag + mov x0, $len + st1 { $acc_l.16b }, [$current_tag] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret .size aes_gcm_dec_kernel,.-aes_gcm_dec_kernel ___ -} -} + # 1. Print directives + print $header_directives; -$code.=<<___; -#endif + # 2. Print Legacy implementation + print get_transformed_code(0, $code_template); + + print "\n"; + + # 3. Print EOR3 implementation + print get_transformed_code(1, $code_template); + + print <<'___'; +#endif // __ARM_MAX_ARCH__ >= 8 ___ -print $code; -close STDOUT or die "error closing STDOUT: $!"; # enforce flush + # Close the handle (which flushes to arm-xlate.pl) + close STDOUT or die "error closing STDOUT: $!"; +} +}
diff --git a/crypto/fipsmodule/aes/gcm.cc.inc b/crypto/fipsmodule/aes/gcm.cc.inc index b0e940a..2dd0e24 100644 --- a/crypto/fipsmodule/aes/gcm.cc.inc +++ b/crypto/fipsmodule/aes/gcm.cc.inc
@@ -152,6 +152,10 @@ if (!len_blocks) { return 0; } + if (impl == gcm_arm64_aes_eor3) { + aes_gcm_enc_kernel_eor3(in, len_blocks * 8, out, Xi, ivec, key, Htable); + return len_blocks; + } aes_gcm_enc_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable); return len_blocks; } @@ -164,6 +168,10 @@ if (!len_blocks) { return 0; } + if (impl == gcm_arm64_aes_eor3) { + aes_gcm_dec_kernel_eor3(in, len_blocks * 8, out, Xi, ivec, key, Htable); + return len_blocks; + } aes_gcm_dec_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable); return len_blocks; } @@ -278,7 +286,10 @@ gcm_key->impl = gcm_x86_aesni; } #elif defined(OPENSSL_AARCH64) - if (gcm_pmull_capable() && is_hwaes) { + // SHA3 and EOR3 belong to the same ISA extension. + if (gcm_sha3_capable() && is_hwaes) { + gcm_key->impl = gcm_arm64_aes_eor3; + } else if (gcm_pmull_capable() && is_hwaes) { gcm_key->impl = gcm_arm64_aes; } #endif @@ -424,7 +435,6 @@ } #if defined(HW_GCM) - // Check |len| to work around a C language bug. See https://crbug.com/1019588. if (key->impl != gcm_separate && len > 0) { // |hw_gcm_encrypt| may not process all the input given to it. It may // not process *any* of its input if it is deemed too small. @@ -513,7 +523,6 @@ } #if defined(HW_GCM) - // Check |len| to work around a C language bug. See https://crbug.com/1019588. if (key->impl != gcm_separate && len > 0) { // |hw_gcm_decrypt| may not process all the input given to it. It may // not process *any* of its input if it is deemed too small.
diff --git a/crypto/fipsmodule/aes/gcm_test.cc b/crypto/fipsmodule/aes/gcm_test.cc index 5a03254..baa4b93 100644 --- a/crypto/fipsmodule/aes/gcm_test.cc +++ b/crypto/fipsmodule/aes/gcm_test.cc
@@ -171,6 +171,20 @@ Htable); } } + if (hwaes_capable() && gcm_pmull_capable() && + CRYPTO_is_ARMv8_SHA3_capable()) { + static const uint8_t kKey[16] = {0}; + uint8_t iv[16] = {0}; + + for (size_t key_bits = 128; key_bits <= 256; key_bits += 64) { + AES_KEY aes_key; + aes_hw_set_encrypt_key(kKey, key_bits, &aes_key); + CHECK_ABI(aes_gcm_enc_kernel_eor3, buf, sizeof(buf) * 8, buf, X, iv, + &aes_key, Htable); + CHECK_ABI(aes_gcm_dec_kernel_eor3, buf, sizeof(buf) * 8, buf, X, iv, + &aes_key, Htable); + } + } #endif } #endif // SUPPORTS_ABI_TEST && !OPENSSL_NO_ASM
diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index 9b42a6a..4723afd 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h
@@ -307,6 +307,7 @@ gcm_x86_vaes_avx2, gcm_x86_vaes_avx512, gcm_arm64_aes, + gcm_arm64_aes_eor3, }; typedef struct { uint64_t hi,lo; } u128; @@ -481,6 +482,7 @@ #define GCM_FUNCREF inline int gcm_pmull_capable() { return CRYPTO_is_ARMv8_PMULL_capable(); } +inline int gcm_sha3_capable() { return CRYPTO_is_ARMv8_SHA3_capable(); } extern "C" void gcm_init_v8(u128 Htable[16], const uint64_t H[2]); extern "C" void gcm_gmult_v8(uint8_t Xi[16], const u128 Htable[16]); @@ -503,6 +505,12 @@ extern "C" void aes_gcm_dec_kernel(const uint8_t *in, uint64_t in_bits, void *out, void *Xi, uint8_t *ivec, const AES_KEY *key, const u128 Htable[16]); +extern "C" void aes_gcm_enc_kernel_eor3(const uint8_t *in, uint64_t in_bits, + void *out, void *Xi, uint8_t *ivec, + const AES_KEY *key, const u128 Htable[16]); +extern "C" void aes_gcm_dec_kernel_eor3(const uint8_t *in, uint64_t in_bits, + void *out, void *Xi, uint8_t *ivec, + const AES_KEY *key, const u128 Htable[16]); #endif #endif
diff --git a/crypto/internal.h b/crypto/internal.h index 287bb6d..2cdb760 100644 --- a/crypto/internal.h +++ b/crypto/internal.h
@@ -1320,6 +1320,9 @@ // ARMV8_SHA512 indicates support for hardware SHA-512 instructions. #define ARMV8_SHA512 (1 << 6) +// ARMV8_SHA3 indicates support for eor3 instructions. +#define ARMV8_SHA3 (1 << 7) + #if defined(OPENSSL_STATIC_ARMCAP) // We assume |CRYPTO_is_*_capable| already checked static capabilities. inline uint32_t OPENSSL_get_armcap() { return 0; } @@ -1404,6 +1407,15 @@ #endif } +inline int CRYPTO_is_ARMv8_SHA3_capable() { + // There is no |OPENSSL_STATIC_ARMCAP_SHA3|. +#if defined(__ARM_FEATURE_SHA3) + return 1; +#else + return (OPENSSL_get_armcap() & ARMV8_SHA3) != 0; +#endif +} + #endif // OPENSSL_ARM || OPENSSL_AARCH64
diff --git a/gen/bcm/aesv8-gcm-armv8-apple.S b/gen/bcm/aesv8-gcm-armv8-apple.S index a0b8e7d..6a76daa 100644 --- a/gen/bcm/aesv8-gcm-armv8-apple.S +++ b/gen/bcm/aesv8-gcm-armv8-apple.S
@@ -5,8 +5,8 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__APPLE__) #if __ARM_MAX_ARCH__ >= 8 - .arch_extension crypto +.arch_extension sha3 .text .globl _aes_gcm_enc_kernel @@ -15,765 +15,731 @@ .align 4 _aes_gcm_enc_kernel: AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! + stp x29, x30, [sp, #-224]! mov x29, sp + ld1 { v0.16b}, [x4] // Load initial counter block stp x19, x20, [sp, #16] - mov x16, x4 - mov x8, x5 + mov v1.16b, v0.16b // Initialize ctr1-3 from ctr0 + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 // Pointer to counter block in memory + mov x8, x5 // Pointer to AES key schedule context stp x21, x22, [sp, #32] - stp x23, x24, [sp, #48] - stp d8, d9, [sp, #64] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel + stp d8, d9, [sp, #64] // Save Neon registers stp d10, d11, [sp, #80] stp d12, d13, [sp, #96] stp d14, d15, [sp, #112] - ldr w17, [x8, #240] - add x19, x8, x17, lsl #4 // borrow input_l1 for last key - ldp x13, x14, [x19] // load round N keys - ldr q31, [x19, #-16] // load round N-1 keys - add x4, x0, x1, lsr #3 // end_input_ptr - lsr x5, x1, #3 // byte_len + ldr w17, [x8, #240] // Load number of AES rounds + add x7, x8, x17, lsl #4 // Calculate pointer to the last round key + ldp x13, x14, [x7] // load round N key (for final XOR) + ldr q31, [x7, #-16] // load round N-1 key + add x4, x0, x1, lsr #3 // Calculate end of input + lsr x5, x1, #3 // Total byte length mov x15, x5 - ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 - ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible - sub x5, x5, #1 // byte_len - 1 - ldr q18, [x8, #0] // load rk0 - and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - ldr q25, [x8, #112] // load rk7 + ldr w12, [x16, #12] // Load counter's low 32 bits + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes add x5, x5, x0 - lsr x12, x11, #32 - fmov d2, x10 // CTR block 2 - orr w11, w11, w11 - rev w12, w12 // rev_ctr32 - fmov d1, x10 // CTR block 1 + rev w12, w12 // Reverse for big-endian increment + uxtw x10, w12 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov x21, #0xc200000000000000 // GHASH reduction constant + str x21, [sp, #128] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., 160 + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str q0, [sp, #160] // Store base counter for block 0-3 + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev w20, w12 + mov v0.s[3], w20 // ctr0 + 0 + add w20, w12, #1 + rev w20, w20 + mov v1.s[3], w20 // ctr0 + 1 + add w20, w12, #2 + rev w20, w20 + mov v2.s[3], w20 // ctr0 + 2 + add w20, w12, #3 + rev w20, w20 + mov v3.s[3], w20 // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add w20, w12, #4 + rev w20, w20 + str w20, [sp, #172] // ctr0 + 4 for next iter + add w20, w12, #5 + rev w20, w20 + str w20, [sp, #188] // ctr0 + 5 for next iter + add w20, w12, #6 + rev w20, w20 + str w20, [sp, #204] // ctr0 + 6 for next iter + add w20, w12, #7 + rev w20, w20 + str w20, [sp, #220] // ctr0 + 7 for next iter + add w12, w12, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 0 - round 0 - add w12, w12, #1 // increment rev_ctr32 - rev w9, w12 // CTR block 1 - fmov d3, x10 // CTR block 3 - orr x9, x11, x9, lsl #32 // CTR block 1 - add w12, w12, #1 // CTR block 1 - ldr q19, [x8, #16] // load rk1 - fmov v1.d[1], x9 // CTR block 1 - rev w9, w12 // CTR block 2 - add w12, w12, #1 // CTR block 2 - orr x9, x11, x9, lsl #32 // CTR block 2 - ldr q20, [x8, #32] // load rk2 - fmov v2.d[1], x9 // CTR block 2 - rev w9, w12 // CTR block 3 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 0 - round 1 - orr x9, x11, x9, lsl #32 // CTR block 3 - fmov v3.d[1], x9 // CTR block 3 + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 1 - round 0 - ldr q21, [x8, #48] // load rk3 - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 0 - round 2 - ldr q24, [x8, #96] // load rk6 + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 2 - round 0 - ldr q23, [x8, #80] // load rk5 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 1 - round 1 - ldr q14, [x6, #48] // load h3l | h3h - ext v14.16b, v14.16b, v14.16b, #8 + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 3 - round 0 + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load H2, H3 (GHASH keys) + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 2 - round 1 - ldr q22, [x8, #64] // load rk4 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 1 - round 2 - ldr q13, [x6, #32] // load h2l | h2h - ext v13.16b, v13.16b, v13.16b, #8 + aesmc v2.16b, v2.16b // AES block 2 - round 1 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 3 - round 1 - ldr q30, [x8, #192] // load rk12 + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 // Byte swap H3 for GHASH + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 // Byte swap H2 for GHASH aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 2 - round 2 - ldr q15, [x6, #80] // load h4l | h4h - ext v15.16b, v15.16b, v15.16b, #8 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 1 - round 3 - ldr q29, [x8, #176] // load rk11 + aesmc v2.16b, v2.16b // AES block 2 - round 2 + ldr q15, [x6, #80] // load H4 aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 3 - round 2 - ldr q26, [x8, #128] // load rk8 - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 2 - round 3 - add w12, w12, #1 // CTR block 3 + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 // Byte swap H4 for GHASH aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 0 - round 3 + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] // Load initial GHASH accumulator (T) + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap T for GHASH + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b // Correct byte order within 64-bit lanes aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 3 - round 3 - ld1 { v11.16b}, [x3] - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 2 - round 4 + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // Karatsuba key: H4_low | H3_low aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 0 - round 4 + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load H1 aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 1 - round 4 + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 // Byte swap H1 for GHASH + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // Karatsuba key: H4_high | H3_high aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 3 - round 4 - cmp x17, #12 // setup flags for AES-128/192/256 check + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // Karatsuba key: H2_low | H1_low aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 0 - round 5 + aesmc v0.16b, v0.16b // AES block 0 - round 5 + ldr q30, [x7] // Preload round N key for final EOR aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 1 - round 5 + aesmc v1.16b, v1.16b // AES block 1 - round 5 aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 3 - round 5 + aesmc v3.16b, v3.16b // AES block 3 - round 5 aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 2 - round 5 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 1 - round 6 - trn2 v17.2d, v14.2d, v15.2d // h4l | h3l - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 3 - round 6 - ldr q27, [x8, #144] // load rk9 + aesmc v2.16b, v2.16b // AES block 2 - round 5 aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 0 - round 6 - ldr q12, [x6] // load h1l | h1h - ext v12.16b, v12.16b, v12.16b, #8 + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 2 - round 6 - ldr q28, [x8, #160] // load rk10 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 1 - round 7 - trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 0 - round 7 + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 2 - round 7 + aesmc v2.16b, v2.16b // AES block 2 - round 7 aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 3 - round 7 - trn2 v16.2d, v12.2d, v13.2d // h2l | h1l - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 1 - round 8 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 2 - round 8 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 3 - round 8 + aesmc v3.16b, v3.16b // AES block 3 - round 7 aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 0 - round 8 - b.lt Lenc_finish_first_blocks // branch if AES-128 - + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_finish_first_blocks // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 1 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 2 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 9 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 3 - round 9 + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 0 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 1 - round 10 + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 2 - round 10 + aesmc v2.16b, v2.16b // AES block 2 - round 10 aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 3 - round 10 + aesmc v3.16b, v3.16b // AES block 3 - round 10 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 0 - round 10 - b.eq Lenc_finish_first_blocks // branch if AES-192 - - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 1 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 2 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 0 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 3 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 1 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 2 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 0 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 3 - round 12 - + aesmc v0.16b, v0.16b // AES block 0 - round 10 + b.eq Lenc_finish_first_blocks // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 Lenc_finish_first_blocks: - cmp x0, x5 // check if we have <= 4 blocks - eor v17.16b, v17.16b, v9.16b // h4k | h3k - aese v2.16b, v31.16b // AES block 2 - round N-1 - trn1 v8.2d, v12.2d, v13.2d // h2h | h1h - aese v1.16b, v31.16b // AES block 1 - round N-1 - aese v0.16b, v31.16b // AES block 0 - round N-1 - aese v3.16b, v31.16b // AES block 3 - round N-1 - eor v16.16b, v16.16b, v8.16b // h2k | h1k - b.ge Lenc_tail // handle tail - - ldp x19, x20, [x0, #16] // AES block 1 - load plaintext - rev w9, w12 // CTR block 4 - ldp x6, x7, [x0, #0] // AES block 0 - load plaintext - ldp x23, x24, [x0, #48] // AES block 3 - load plaintext - ldp x21, x22, [x0, #32] // AES block 2 - load plaintext - add x0, x0, #64 // AES input_ptr update - eor x19, x19, x13 // AES block 1 - round N low - eor x20, x20, x14 // AES block 1 - round N high - fmov d5, x19 // AES block 1 - mov low - eor x6, x6, x13 // AES block 0 - round N low - eor x7, x7, x14 // AES block 0 - round N high - eor x24, x24, x14 // AES block 3 - round N high - fmov d4, x6 // AES block 0 - mov low - cmp x0, x5 // check if we have <= 8 blocks - fmov v4.d[1], x7 // AES block 0 - mov high - eor x23, x23, x13 // AES block 3 - round N low - eor x21, x21, x13 // AES block 2 - round N low - fmov v5.d[1], x20 // AES block 1 - mov high - fmov d6, x21 // AES block 2 - mov low - add w12, w12, #1 // CTR block 4 - orr x9, x11, x9, lsl #32 // CTR block 4 - fmov d7, x23 // AES block 3 - mov low - eor x22, x22, x14 // AES block 2 - round N high - fmov v6.d[1], x22 // AES block 2 - mov high - eor v4.16b, v4.16b, v0.16b // AES block 0 - result - fmov d0, x10 // CTR block 4 - fmov v0.d[1], x9 // CTR block 4 - rev w9, w12 // CTR block 5 - add w12, w12, #1 // CTR block 5 - eor v5.16b, v5.16b, v1.16b // AES block 1 - result - fmov d1, x10 // CTR block 5 - orr x9, x11, x9, lsl #32 // CTR block 5 - fmov v1.d[1], x9 // CTR block 5 - rev w9, w12 // CTR block 6 - st1 { v4.16b}, [x2], #16 // AES block 0 - store result - fmov v7.d[1], x24 // AES block 3 - mov high - orr x9, x11, x9, lsl #32 // CTR block 6 - eor v6.16b, v6.16b, v2.16b // AES block 2 - result - st1 { v5.16b}, [x2], #16 // AES block 1 - store result - add w12, w12, #1 // CTR block 6 - fmov d2, x10 // CTR block 6 - fmov v2.d[1], x9 // CTR block 6 - st1 { v6.16b}, [x2], #16 // AES block 2 - store result - rev w9, w12 // CTR block 7 - orr x9, x11, x9, lsl #32 // CTR block 7 - eor v7.16b, v7.16b, v3.16b // AES block 3 - result - st1 { v7.16b}, [x2], #16 // AES block 3 - store result - b.ge Lenc_prepretail // do prepretail - -Lenc_main_loop: // main loop start + cmp x0, x5 // check if we have <= 4 blocks to process in the tail + eor v17.16b, v17.16b, v9.16b // Karatsuba key: H3^H4 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // Karatsuba key: H2_high | H1_high + eor v16.16b, v16.16b, v8.16b // Karatsuba key: H1^H2 + b.ge Lenc_tail // handle tail if no more full 4-block sets + ldp q6, q7, [x0, #32] // AES blocks 2,3 load plaintext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + eor v4.16b, v4.16b, v30.16b + eor v4.16b, v4.16b, v0.16b // AES block 0 - result = PT ^ AES(ctr0) + eor v5.16b, v5.16b, v30.16b + eor v5.16b, v5.16b, v1.16b // AES block 1 - result = PT ^ AES(ctr1) + eor v6.16b, v6.16b, v30.16b + eor v6.16b, v6.16b, v2.16b // AES block 2 - result = PT ^ AES(ctr2) + eor v7.16b, v7.16b, v30.16b + eor v7.16b, v7.16b, v3.16b // AES block 3 - result = PT ^ AES(ctr3) + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // Prepare and store counter values for the third iteration + rev w20, w12 + str w20, [sp, #172] // ctr + 8 + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] // ctr + 9 + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] // ctr + 10 + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] // ctr + 11 + add w12, w12, #4 // Advance counter base + cmp x0, x5 // check if we have <= 4 blocks remaining + b.ge Lenc_prepretail // go to prepretail if < 2 full loops left +Lenc_main_loop: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d3, x10 // CTR block 4k+3 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - fmov v3.d[1], x9 // CTR block 4k+3 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - ldp x23, x24, [x0, #48] // AES block 4k+7 - load plaintext - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - ldp x21, x22, [x0, #32] // AES block 4k+6 - load plaintext - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - eor v4.16b, v4.16b, v11.16b // PRE 1 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - eor x23, x23, x13 // AES block 4k+7 - round N low - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - mov d10, v17.d[1] // GHASH block 4k - mid - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - eor x22, x22, x14 // AES block 4k+6 - round N high - mov d8, v4.d[1] // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + ldr d8, [sp, #128] // Load GHASH reduction constant + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 v4.16b, v4.16b // GHASH block 4k - Byte swap CT + rev64 v5.16b, v5.16b // GHASH block 4k+1 - Byte swap CT + rev64 v6.16b, v6.16b // GHASH block 4k+2 - Byte swap CT + rev64 v7.16b, v7.16b // GHASH block 4k+3 - Byte swap CT + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // GHASH - prepare acc for XOR + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid Karatsuba key + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d20, v4.d[1] // GHASH block 4k - mid aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - mov d8, v6.d[1] // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v20.8b, v20.8b, v4.8b // GHASH block 4k - mid aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + mov d21, v5.d[1] // GHASH block 4k+1 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + eor v21.8b, v21.8b, v5.8b // GHASH block 4k+1 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + pmull v10.1q, v20.1d, v10.1d // GHASH block 4k - mid aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + pmull v21.1q, v21.1d, v17.1d // GHASH block 4k+1 - mid aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + eor v10.16b, v10.16b, v21.16b // GHASH block 4k+1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + ext v22.16b, v22.16b, v6.16b, #8 // GHASH block 4k+2 - mid aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v22.16b, v22.16b, v6.16b // GHASH block 4k+2 - mid aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull2 v22.1q, v22.2d, v16.2d // GHASH block 4k+2 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + mov d23, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v23.8b, v23.8b, v7.8b // GHASH block 4k+3 - mid aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - ldp x19, x20, [x0, #16] // AES block 4k+5 - load plaintext - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - mov d4, v7.d[1] // GHASH block 4k+3 - mid + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + pmull v23.1q, v23.1d, v16.1d // GHASH block 4k+3 - mid aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v10.16b, v10.16b, v22.16b + eor v10.16b, v10.16b, v23.16b // GHASH block 4k+2/3 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v22.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v21.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v22.16b, v22.16b, v21.16b // GHASH block 4k+3 - high + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull2 v23.1q, v5.2d, v14.2d // GHASH block 4k+1 - high aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor x19, x19, x13 // AES block 4k+5 - round N low - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + pmull v21.1q, v6.1d, v13.1d // GHASH block 4k+2 - low aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - eor x21, x21, x13 // AES block 4k+6 - round N low + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull v20.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + eor v9.16b, v9.16b, v22.16b + eor v9.16b, v9.16b, v23.16b // GHASH block 4k/1/2/3 - high + pmull v22.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + ldp q6, q7, [x0, #32] + ldp q4, q5, [x0], #64 + eor v20.16b, v20.16b, v21.16b // GHASH block 4k+1 - low + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + eor v11.16b, v11.16b, v22.16b + eor v11.16b, v11.16b, v20.16b // GHASH block 4k/1/2/3 - low + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + eor v10.16b, v10.16b, v9.16b + eor v10.16b, v10.16b, v11.16b // MODULO - karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v20.1q, v9.1d, v8.1d // MODULO - top 64b align with mid aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - movi v8.8b, #0xc2 - pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - cmp x17, #12 // setup flags for AES-128/192/256 check - fmov d5, x19 // AES block 4k+5 - mov low - ldp x6, x7, [x0, #0] // AES block 4k+4 - load plaintext - b.lt Lenc_main_loop_continue // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_main_loop_continue // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - b.eq Lenc_main_loop_continue // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Lenc_main_loop_continue // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 Lenc_main_loop_continue: - shl d8, d8, #56 // mod_constant - eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid - add w12, w12, #1 // CTR block 4k+3 - eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - add x0, x0, #64 // AES input_ptr update - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - rev w9, w12 // CTR block 4k+8 - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor x6, x6, x13 // AES block 4k+4 - round N low - eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up - eor x7, x7, x14 // AES block 4k+4 - round N high - fmov d4, x6 // AES block 4k+4 - mov low - orr x9, x11, x9, lsl #32 // CTR block 4k+8 - eor v7.16b, v9.16b, v7.16b // MODULO - fold into mid - eor x20, x20, x14 // AES block 4k+5 - round N high - eor x24, x24, x14 // AES block 4k+7 - round N high - add w12, w12, #1 // CTR block 4k+8 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - fmov v4.d[1], x7 // AES block 4k+4 - mov high - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - fmov d7, x23 // AES block 4k+7 - mov low - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - fmov v5.d[1], x20 // AES block 4k+5 - mov high - fmov d6, x21 // AES block 4k+6 - mov low - cmp x0, x5 // LOOP CONTROL - fmov v6.d[1], x22 // AES block 4k+6 - mov high - pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result - fmov d0, x10 // CTR block 4k+8 - fmov v0.d[1], x9 // CTR block 4k+8 - rev w9, w12 // CTR block 4k+9 - add w12, w12, #1 // CTR block 4k+9 - eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result - fmov d1, x10 // CTR block 4k+9 - orr x9, x11, x9, lsl #32 // CTR block 4k+9 - fmov v1.d[1], x9 // CTR block 4k+9 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - rev w9, w12 // CTR block 4k+10 - st1 { v4.16b}, [x2], #16 // AES block 4k+4 - store result - orr x9, x11, x9, lsl #32 // CTR block 4k+10 - eor v11.16b, v11.16b, v9.16b // MODULO - fold into low - fmov v7.d[1], x24 // AES block 4k+7 - mov high - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - st1 { v5.16b}, [x2], #16 // AES block 4k+5 - store result - add w12, w12, #1 // CTR block 4k+10 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result - fmov d2, x10 // CTR block 4k+10 - st1 { v6.16b}, [x2], #16 // AES block 4k+6 - store result - fmov v2.d[1], x9 // CTR block 4k+10 - rev w9, w12 // CTR block 4k+11 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - orr x9, x11, x9, lsl #32 // CTR block 4k+11 - eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result - st1 { v7.16b}, [x2], #16 // AES block 4k+7 - store result + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v20.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v20.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v9.16b, v11.16b + eor v11.16b, v11.16b, v20.16b // MODULO - fold into low + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor v4.16b, v4.16b, v30.16b + eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor v5.16b, v5.16b, v30.16b + eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor v6.16b, v6.16b, v30.16b + eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor v7.16b, v7.16b, v30.16b + eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // We used these registers as temporaries above so reload the RKs. + ldp q20, q21, [x8, #32] // load rk2, rk3 + ldp q22, q23, [x8, #64] // load rk4, rk5 + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + rev w20, w12 + str w20, [sp, #172] + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] + add w12, w12, #4 + cmp x0, x5 // LOOP CONTROL b.lt Lenc_main_loop - Lenc_prepretail: // PREPRETAIL aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) + aesmc v1.16b, v1.16b // AES block 1 - round 0 + rev64 v6.16b, v6.16b // GHASH block 2 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - fmov d3, x10 // CTR block 4k+3 + aesmc v2.16b, v2.16b // AES block 2 - round 0 aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) - fmov v3.d[1], x9 // CTR block 4k+3 - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aesmc v0.16b, v0.16b // AES block 0 - round 0 + rev64 v4.16b, v4.16b // GHASH block 0 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aesmc v2.16b, v2.16b // AES block 2 - round 1 aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v5.16b, v5.16b // GHASH block 1 aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + aesmc v2.16b, v2.16b // AES block 2 - round 2 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - mov d10, v17.d[1] // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 3 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid Karatsuba key aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - mov d8, v4.d[1] // GHASH block 4k - mid - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + aesmc v2.16b, v2.16b // AES block 2 - round 3 aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + aesmc v0.16b, v0.16b // AES block 0 - round 2 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + aesmc v3.16b, v3.16b // AES block 3 - round 1 aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aesmc v1.16b, v1.16b // AES block 1 - round 3 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - mov d4, v5.d[1] // GHASH block 4k+1 - mid + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + mov d4, v5.d[1] // GHASH block 1 - mid aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low + aesmc v0.16b, v0.16b // AES block 0 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - mov d8, v6.d[1] // GHASH block 4k+2 - mid + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + mov d8, v6.d[1] // GHASH block 2 - mid aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) + aesmc v0.16b, v0.16b // AES block 0 - round 4 + rev64 v7.16b, v7.16b // GHASH block 3 aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - add w12, w12, #1 // CTR block 4k+3 - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aesmc v3.16b, v3.16b // AES block 3 - round 5 aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 2 - round 4 + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + ins v8.d[1], v8.d[0] // GHASH block 2 - mid aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - mov d4, v7.d[1] // GHASH block 4k+3 - mid + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v9.16b, v9.16b, v4.16b // GHASH block 2 - high + mov d4, v7.d[1] // GHASH block 3 - mid aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + eor v4.8b, v4.8b, v7.8b // GHASH block 3 - mid + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid + aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull v4.1q, v4.1d, v16.1d // GHASH block 3 - mid + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + aesmc v0.16b, v0.16b // AES block 0 - round 5 aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + aesmc v1.16b, v1.16b // AES block 1 - round 6 aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + aesmc v2.16b, v2.16b // AES block 2 - round 6 aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - movi v8.8b, #0xc2 + aesmc v0.16b, v0.16b // AES block 0 - round 6 aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + aesmc v3.16b, v3.16b // AES block 3 - round 6 aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high + aesmc v1.16b, v1.16b // AES block 1 - round 7 + eor v9.16b, v9.16b, v5.16b // GHASH block 3 - high aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + aesmc v0.16b, v0.16b // AES block 0 - round 7 aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - shl d8, d8, #56 // mod_constant + aesmc v3.16b, v3.16b // AES block 3 - round 7 + ldr d8, [sp, #128] aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid - pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aesmc v1.16b, v1.16b // AES block 1 - round 8 + eor v10.16b, v10.16b, v4.16b // GHASH block 3 - mid + pmull v6.1q, v7.1d, v12.1d // GHASH block 3 - low aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - cmp x17, #12 // setup flags for AES-128/192/256 check + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low + aesmc v0.16b, v0.16b // AES block 0 - round 8 + eor v11.16b, v11.16b, v6.16b // GHASH block 3 - low aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v10.16b, v10.16b, v9.16b // karatsuba tidy up + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v10.16b, v10.16b, v9.16b // karatsuba tidy up aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + aesmc v2.16b, v2.16b // AES block 2 - round 8 pmull v4.1q, v9.1d, v8.1d ext v9.16b, v9.16b, v9.16b, #8 eor v10.16b, v10.16b, v11.16b - b.lt Lenc_finish_prepretail // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + b.lt Lenc_finish_prepretail // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - b.eq Lenc_finish_prepretail // branch if AES-192 - - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Lenc_finish_prepretail // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 Lenc_finish_prepretail: + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 eor v10.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v9.16b pmull v4.1q, v10.1d, v8.1d ext v10.16b, v10.16b, v10.16b, #8 - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 eor v11.16b, v11.16b, v4.16b - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 eor v11.16b, v11.16b, v10.16b - -Lenc_tail: // TAIL - ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag - sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process - ldp x6, x7, [x0], #16 // AES block 4k+4 - load plaintext - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high +Lenc_tail: // TAIL: Process remaining 0 to 3 blocks + ext v8.16b, v11.16b, v11.16b, #8 // Save current GHASH state for partial tag feed-in + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ldp x6, x7, [x0], #16 // AES block 0 - load plaintext + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high cmp x5, #48 - fmov d4, x6 // AES block 4k+4 - mov low - fmov v4.d[1], x7 // AES block 4k+4 - mov high - eor v5.16b, v4.16b, v0.16b // AES block 4k+4 - result + fmov d4, x6 // AES block 0 - mov low + fmov v4.d[1], x7 // AES block 0 - mov high + eor v5.16b, v4.16b, v0.16b // AES block 0 - result b.gt Lenc_blocks_more_than_3 cmp x5, #32 mov v3.16b, v2.16b movi v11.8b, #0 movi v9.8b, #0 - sub w12, w12, #1 mov v2.16b, v1.16b movi v10.8b, #0 b.gt Lenc_blocks_more_than_2 mov v3.16b, v1.16b - sub w12, w12, #1 cmp x5, #16 b.gt Lenc_blocks_more_than_1 - sub w12, w12, #1 b Lenc_blocks_less_than_1 Lenc_blocks_more_than_3: // blocks left > 3 - st1 { v5.16b}, [x2], #16 // AES final-3 block - store result - ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high - rev64 v4.16b, v5.16b // GHASH final-3 block - eor x6, x6, x13 // AES final-2 block - round N low - eor v4.16b, v4.16b, v8.16b // feed in partial tag - eor x7, x7, x14 // AES final-2 block - round N high - mov d22, v4.d[1] // GHASH final-3 block - mid - fmov d5, x6 // AES final-2 block - mov low - fmov v5.d[1], x7 // AES final-2 block - mov high - eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid - movi v8.8b, #0 // suppress further partial tag feed in - mov d10, v17.d[1] // GHASH final-3 block - mid - pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low - pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high - pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid - eor v5.16b, v5.16b, v1.16b // AES final-2 block - result + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-3 block + eor x6, x6, x13 // AES final-2 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor x7, x7, x14 // AES final-2 block - round N high + mov d22, v4.d[1] // GHASH final-3 block - mid + fmov d5, x6 // AES final-2 block - mov low + fmov v5.d[1], x7 // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + mov d10, v17.d[1] // GHASH final-3 block - mid + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor v5.16b, v5.16b, v1.16b // AES final-2 block - result Lenc_blocks_more_than_2: // blocks left > 2 - st1 { v5.16b}, [x2], #16 // AES final-2 block - store result - ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high - rev64 v4.16b, v5.16b // GHASH final-2 block - eor x6, x6, x13 // AES final-1 block - round N low - eor v4.16b, v4.16b, v8.16b // feed in partial tag - fmov d5, x6 // AES final-1 block - mov low - eor x7, x7, x14 // AES final-1 block - round N high - fmov v5.d[1], x7 // AES final-1 block - mov high - movi v8.8b, #0 // suppress further partial tag feed in - pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high - mov d22, v4.d[1] // GHASH final-2 block - mid - pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low - eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid - eor v5.16b, v5.16b, v2.16b // AES final-1 block - result - eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high - pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low - eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-2 block + eor x6, x6, x13 // AES final-1 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + fmov d5, x6 // AES final-1 block - mov low + eor x7, x7, x14 // AES final-1 block - round N high + fmov v5.d[1], x7 // AES final-1 block - mov high + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + eor v5.16b, v5.16b, v2.16b // AES final-1 block - result + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid Lenc_blocks_more_than_1: // blocks left > 1 - st1 { v5.16b}, [x2], #16 // AES final-1 block - store result - rev64 v4.16b, v5.16b // GHASH final-1 block - ldp x6, x7, [x0], #16 // AES final block - load input low & high - eor v4.16b, v4.16b, v8.16b // feed in partial tag - movi v8.8b, #0 // suppress further partial tag feed in - eor x6, x6, x13 // AES final block - round N low - mov d22, v4.d[1] // GHASH final-1 block - mid - pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high - eor x7, x7, x14 // AES final block - round N high - eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high - ins v22.d[1], v22.d[0] // GHASH final-1 block - mid - fmov d5, x6 // AES final block - mov low - fmov v5.d[1], x7 // AES final block - mov high - pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid - pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low - eor v5.16b, v5.16b, v3.16b // AES final block - result - eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low -Lenc_blocks_less_than_1: // blocks left <= 1 - and x1, x1, #127 // bit_length %= 128 - mvn x13, xzr // rkN_l = 0xffffffffffffffff - sub x1, x1, #128 // bit_length -= 128 - neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) - ld1 { v18.16b}, [x2] // load existing bytes where the possibly partial last block is to be stored - mvn x14, xzr // rkN_h = 0xffffffffffffffff - and x1, x1, #127 // bit_length %= 128 - lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + st1 { v5.16b}, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block: Byte Swap CT + ldp x6, x7, [x0], #16 // AES final block - load plaintext + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + movi v8.8b, #0 // Clear for next block + eor x6, x6, x13 // AES final block - round N low + mov d22, v4.d[1] // GHASH final-1 block - mid + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor x7, x7, x14 // AES final block - round N high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + fmov d5, x6 // AES final block - mov low + fmov v5.d[1], x7 // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + eor v5.16b, v5.16b, v3.16b // AES final block - result + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low +Lenc_blocks_less_than_1: // Last partial block handling + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x13, xzr // Mask for low 64 bits + sub x1, x1, #128 // + neg x1, x1 // Valid bits in the last block (1-128) + ldr q18, [x2] // Load destination for merging + mvn x14, xzr // Mask for high 64 bits + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block cmp x1, #64 csel x6, x13, x14, lt csel x7, x14, xzr, lt - fmov d0, x6 // ctr0b is mask for last block + fmov d0, x6 // ctr0d is mask for last block fmov v0.d[1], x7 - and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits - rev64 v4.16b, v5.16b // GHASH final block - eor v4.16b, v4.16b, v8.16b // feed in partial tag - bif v5.16b, v18.16b, v0.16b // insert existing bytes in top end of result before storing - pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high - mov d8, v4.d[1] // GHASH final block - mid - rev w9, w12 - pmull v21.1q, v4.1d, v12.1d // GHASH final block - low - eor v9.16b, v9.16b, v20.16b // GHASH final block - high - eor v8.8b, v8.8b, v4.8b // GHASH final block - mid - pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final block - low - eor v10.16b, v10.16b, v8.16b // GHASH final block - mid - movi v8.8b, #0xc2 - eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - shl d8, d8, #56 // mod_constant - eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - str w9, [x16, #12] // store the updated counter - st1 { v5.16b}, [x2] // store all 16B - eor v11.16b, v11.16b, v9.16b // MODULO - fold into low - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + and v5.16b, v5.16b, v0.16b // Mask out unused bits of the last CT block + rev64 v4.16b, v5.16b // GHASH final block - byte swap + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + bif v5.16b, v18.16b, v0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + mov d8, v4.d[1] // GHASH final block - mid + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + fmov d8, x21 + eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + st1 { v5.16b}, [x2] // store all 16B + eor v11.16b, v11.16b, v9.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap GHASH result + rev64 v11.16b, v11.16b // Final Tag mov x0, x15 - st1 { v11.16b }, [x3] + st1 { v11.16b }, [x3] // Store final tag ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] ldp d8, d9, [sp, #64] ldp d10, d11, [sp, #80] ldp d12, d13, [sp, #96] ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 + ldp x29, x30, [sp], #224 AARCH64_VALIDATE_LINK_REGISTER ret @@ -783,9 +749,13 @@ .align 4 _aes_gcm_dec_kernel: AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! + stp x29, x30, [sp, #-224]! mov x29, sp stp x19, x20, [sp, #16] + ld1 { v0.16b}, [x4] + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b mov x16, x4 mov x8, x5 stp x21, x22, [sp, #32] @@ -794,752 +764,705 @@ stp d10, d11, [sp, #80] stp d12, d13, [sp, #96] stp d14, d15, [sp, #112] - ldr w17, [x8, #240] - add x19, x8, x17, lsl #4 // borrow input_l1 for last key - ldp x13, x14, [x19] // load round N keys - ldr q31, [x19, #-16] // load round N-1 keys - lsr x5, x1, #3 // byte_len + ldr w17, [x8, #240] // Load number of AES rounds + add x19, x8, x17, lsl #4 // borrow input_l1 for last key + ldp x13, x14, [x19] // load round N keys + ldr q31, [x19, #-16] // load round N-1 keys + add x4, x0, x1, lsr #3 // end_input_ptr + lsr x5, x1, #3 // byte_len mov x15, x5 - ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 - ldr q26, [x8, #128] // load rk8 - sub x5, x5, #1 // byte_len - 1 - ldr q25, [x8, #112] // load rk7 - and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - add x4, x0, x1, lsr #3 // end_input_ptr - ldr q24, [x8, #96] // load rk6 - lsr x12, x11, #32 - ldr q23, [x8, #80] // load rk5 - orr w11, w11, w11 - ldr q21, [x8, #48] // load rk3 + ldr w9, [x16, #12] // Load scalar 32-bit counter (CTR) + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) add x5, x5, x0 - rev w12, w12 // rev_ctr32 - add w12, w12, #1 // increment rev_ctr32 - fmov d3, x10 // CTR block 3 - rev w9, w12 // CTR block 1 - add w12, w12, #1 // CTR block 1 - fmov d1, x10 // CTR block 1 - orr x9, x11, x9, lsl #32 // CTR block 1 - ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible - fmov v1.d[1], x9 // CTR block 1 - rev w9, w12 // CTR block 2 - add w12, w12, #1 // CTR block 2 - fmov d2, x10 // CTR block 2 - orr x9, x11, x9, lsl #32 // CTR block 2 - fmov v2.d[1], x9 // CTR block 2 - rev w9, w12 // CTR block 3 - orr x9, x11, x9, lsl #32 // CTR block 3 - ldr q18, [x8, #0] // load rk0 - fmov v3.d[1], x9 // CTR block 3 - add w12, w12, #1 // CTR block 3 - ldr q22, [x8, #64] // load rk4 - ldr q19, [x8, #16] // load rk1 + rev w9, w9 // Reverse it once for big-endian incrementing + uxtw x10, w9 // Zero extend reversed w9 into x10 + str q0, [sp, #160] + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + rev w20, w9 + mov v0.s[3], w20 + add w20, w9, #1 + rev w20, w20 + mov v1.s[3], w20 + add w20, w9, #2 + rev w20, w20 + mov v2.s[3], w20 + add w20, w9, #3 + rev w20, w20 + mov v3.s[3], w20 + add w20, w9, #4 + rev w20, w20 + str w20, [sp, #172] + add w20, w9, #5 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #6 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #7 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov x21, #0xc200000000000000 + str x21, [sp, #128] + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load h2, h3 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 + ldr q15, [x6, #80] // load h4 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // h4l | h3l + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load h1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // h2l | h1l + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_first_blocks // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_first_blocks // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Ldec_finish_first_blocks: + ldr q27, [x19] // load rkN + cmp x0, x5 // check if we have <= 4 blocks + eor v17.16b, v17.16b, v9.16b // h4k | h3k + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // h2h | h1h + eor v16.16b, v16.16b, v8.16b // h2k | h1k + b.ge Ldec_tail // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value (v30). This avoids several fmovs. + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + eor v0.16b, v4.16b, v0.16b + eor v0.16b, v0.16b, v27.16b // AES block 0 - result + eor v1.16b, v5.16b, v1.16b + eor v1.16b, v1.16b, v27.16b // AES block 1 - result + eor v2.16b, v6.16b, v2.16b + eor v2.16b, v2.16b, v27.16b // AES block 2 - result + eor v3.16b, v7.16b, v3.16b + eor v3.16b, v3.16b, v27.16b // AES block 3 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 0-3 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 // check if we have <= 4 blocks + b.ge Ldec_prepretail // do prepretail +Ldec_main_loop: // main loop start + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + rev64 v4.16b, v4.16b // GHASH block 4k + rev64 v5.16b, v5.16b // GHASH block 4k+1 + rev64 v6.16b, v6.16b // GHASH block 4k+2 + rev64 v7.16b, v7.16b // GHASH block 4k+3 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // PRE 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d8, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + mov d4, v5.d[1] // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v5.16b // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + mov d8, v6.d[1] // GHASH block 4k+2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid + ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v28.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + mov d6, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull v27.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + eor v10.16b, v10.16b, v4.16b + eor v10.16b, v10.16b, v8.16b // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid + eor v9.16b, v9.16b, v28.16b + eor v9.16b, v9.16b, v5.16b // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid + ldr d8, [sp, #128] + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + eor v10.16b, v10.16b, v6.16b + eor v10.16b, v10.16b, v7.16b // GHASH block 4k+3 - mid & MODULO - fold into mid + eor v11.16b, v11.16b, v27.16b // GHASH block 4k+3 - low + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v6.16b + eor v10.16b, v10.16b, v9.16b // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_main_loop_continue // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Ldec_main_loop_continue // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +Ldec_main_loop_continue: + ldr q27, [x19] // load rkN + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor v0.16b, v4.16b, v0.16b + eor v0.16b, v0.16b, v27.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor v1.16b, v5.16b, v1.16b + eor v1.16b, v1.16b, v27.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor v2.16b, v6.16b, v2.16b + eor v2.16b, v2.16b, v27.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor v3.16b, v7.16b, v3.16b + eor v3.16b, v3.16b, v27.16b // AES block 4k+7 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 + b.lt Ldec_main_loop +Ldec_prepretail: // PREPRETAIL + rev64 v4.16b, v4.16b // GHASH block 0 + rev64 v5.16b, v5.16b // GHASH block 1 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 aese v0.16b, v18.16b aesmc v0.16b, v0.16b // AES block 0 - round 0 - ldr q14, [x6, #48] // load h3l | h3h - ext v14.16b, v14.16b, v14.16b, #8 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 3 - round 0 - ldr q15, [x6, #80] // load h4l | h4h - ext v15.16b, v15.16b, v15.16b, #8 aese v1.16b, v18.16b aesmc v1.16b, v1.16b // AES block 1 - round 0 - ldr q13, [x6, #32] // load h2l | h2h - ext v13.16b, v13.16b, v13.16b, #8 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 2 - round 0 - ldr q20, [x8, #32] // load rk2 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v6.16b, v6.16b // GHASH block 2 aese v1.16b, v19.16b aesmc v1.16b, v1.16b // AES block 1 - round 1 - ld1 { v11.16b}, [x3] - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high aese v2.16b, v19.16b aesmc v2.16b, v2.16b // AES block 2 - round 1 - ldr q27, [x8, #144] // load rk9 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low aese v3.16b, v19.16b aesmc v3.16b, v3.16b // AES block 3 - round 1 - ldr q30, [x8, #192] // load rk12 + mov d4, v5.d[1] // GHASH block 1 - mid aese v0.16b, v20.16b aesmc v0.16b, v0.16b // AES block 0 - round 2 - ldr q12, [x6] // load h1l | h1h - ext v12.16b, v12.16b, v12.16b, #8 - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 2 - round 2 - ldr q28, [x8, #160] // load rk10 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 3 - round 2 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 0 - round 3 aese v1.16b, v20.16b aesmc v1.16b, v1.16b // AES block 1 - round 2 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + mov d8, v6.d[1] // GHASH block 2 - mid + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low aese v0.16b, v22.16b aesmc v0.16b, v0.16b // AES block 0 - round 4 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + ins v8.d[1], v8.d[0] // GHASH block 2 - mid aese v2.16b, v21.16b aesmc v2.16b, v2.16b // AES block 2 - round 3 aese v1.16b, v21.16b aesmc v1.16b, v1.16b // AES block 1 - round 3 - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 3 - round 4 + eor v9.16b, v9.16b, v4.16b + eor v9.16b, v9.16b, v5.16b // GHASH block 2 - high & GHASH block 3 - high + pmull v4.1q, v7.1d, v12.1d // GHASH block 3 - low aese v2.16b, v22.16b aesmc v2.16b, v2.16b // AES block 2 - round 4 + mov d6, v7.d[1] // GHASH block 3 - mid aese v1.16b, v22.16b aesmc v1.16b, v1.16b // AES block 1 - round 4 - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 3 - round 5 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 0 - round 5 - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid aese v2.16b, v23.16b aesmc v2.16b, v2.16b // AES block 2 - round 5 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 0 - round 6 + eor v6.8b, v6.8b, v7.8b // GHASH block 3 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 aese v3.16b, v24.16b aesmc v3.16b, v3.16b // AES block 3 - round 6 - cmp x17, #12 // setup flags for AES-128/192/256 check - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid aese v2.16b, v24.16b aesmc v2.16b, v2.16b // AES block 2 - round 6 - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 0 - round 7 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + movi v8.8b, #0xc2 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v11.16b, v11.16b, v4.16b // GHASH block 3 - low + pmull v6.1q, v6.1d, v16.1d // GHASH block 3 - mid aese v3.16b, v25.16b aesmc v3.16b, v3.16b // AES block 3 - round 7 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 0 - round 8 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + eor v10.16b, v10.16b, v6.16b // GHASH block 3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b // AES block 3 - round 8 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up aese v1.16b, v26.16b aesmc v1.16b, v1.16b // AES block 1 - round 8 - ldr q29, [x8, #176] // load rk11 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + shl d8, d8, #56 // mod_constant aese v2.16b, v26.16b aesmc v2.16b, v2.16b // AES block 2 - round 8 - b.lt Ldec_finish_first_blocks // branch if AES-128 - + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_prepretail // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 0 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 1 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 3 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 2 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 0 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 10 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 1 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 3 - round 10 + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 2 - round 10 - b.eq Ldec_finish_first_blocks // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 0 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 3 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 1 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 2 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 1 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 0 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 2 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 3 - round 12 - -Ldec_finish_first_blocks: - cmp x0, x5 // check if we have <= 4 blocks - trn1 v9.2d, v14.2d, v15.2d // h4h | h3h - trn2 v17.2d, v14.2d, v15.2d // h4l | h3l - trn1 v8.2d, v12.2d, v13.2d // h2h | h1h - trn2 v16.2d, v12.2d, v13.2d // h2l | h1l - eor v17.16b, v17.16b, v9.16b // h4k | h3k - aese v1.16b, v31.16b // AES block 1 - round N-1 - aese v2.16b, v31.16b // AES block 2 - round N-1 - eor v16.16b, v16.16b, v8.16b // h2k | h1k - aese v3.16b, v31.16b // AES block 3 - round N-1 - aese v0.16b, v31.16b // AES block 0 - round N-1 - b.ge Ldec_tail // handle tail - - ldr q4, [x0, #0] // AES block 0 - load ciphertext - ldr q5, [x0, #16] // AES block 1 - load ciphertext - rev w9, w12 // CTR block 4 - eor v0.16b, v4.16b, v0.16b // AES block 0 - result - eor v1.16b, v5.16b, v1.16b // AES block 1 - result - rev64 v5.16b, v5.16b // GHASH block 1 - ldr q7, [x0, #48] // AES block 3 - load ciphertext - mov x7, v0.d[1] // AES block 0 - mov high - mov x6, v0.d[0] // AES block 0 - mov low - rev64 v4.16b, v4.16b // GHASH block 0 - add w12, w12, #1 // CTR block 4 - fmov d0, x10 // CTR block 4 - orr x9, x11, x9, lsl #32 // CTR block 4 - fmov v0.d[1], x9 // CTR block 4 - rev w9, w12 // CTR block 5 - add w12, w12, #1 // CTR block 5 - mov x19, v1.d[0] // AES block 1 - mov low - orr x9, x11, x9, lsl #32 // CTR block 5 - mov x20, v1.d[1] // AES block 1 - mov high - eor x7, x7, x14 // AES block 0 - round N high - eor x6, x6, x13 // AES block 0 - round N low - stp x6, x7, [x2], #16 // AES block 0 - store result - fmov d1, x10 // CTR block 5 - ldr q6, [x0, #32] // AES block 2 - load ciphertext - add x0, x0, #64 // AES input_ptr update - fmov v1.d[1], x9 // CTR block 5 - rev w9, w12 // CTR block 6 - add w12, w12, #1 // CTR block 6 - eor x19, x19, x13 // AES block 1 - round N low - orr x9, x11, x9, lsl #32 // CTR block 6 - eor x20, x20, x14 // AES block 1 - round N high - stp x19, x20, [x2], #16 // AES block 1 - store result - eor v2.16b, v6.16b, v2.16b // AES block 2 - result - cmp x0, x5 // check if we have <= 8 blocks - b.ge Ldec_prepretail // do prepretail - -Ldec_main_loop: // main loop start - mov x21, v2.d[0] // AES block 4k+2 - mov low - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - mov x22, v2.d[1] // AES block 4k+2 - mov high - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d2, x10 // CTR block 4k+6 - fmov v2.d[1], x9 // CTR block 4k+6 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev w9, w12 // CTR block 4k+7 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - mov x24, v3.d[1] // AES block 4k+3 - mov high - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - mov x23, v3.d[0] // AES block 4k+3 - mov low - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - mov d8, v4.d[1] // GHASH block 4k - mid - fmov d3, x10 // CTR block 4k+7 - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - orr x9, x11, x9, lsl #32 // CTR block 4k+7 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - fmov v3.d[1], x9 // CTR block 4k+7 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - eor x22, x22, x14 // AES block 4k+2 - round N high - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - mov d10, v17.d[1] // GHASH block 4k - mid - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - rev64 v6.16b, v6.16b // GHASH block 4k+2 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - eor x21, x21, x13 // AES block 4k+2 - round N low - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - stp x21, x22, [x2], #16 // AES block 4k+2 - store result - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - rev64 v7.16b, v7.16b // GHASH block 4k+3 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - eor x23, x23, x13 // AES block 4k+3 - round N low - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - eor x24, x24, x14 // AES block 4k+3 - round N high - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - add w12, w12, #1 // CTR block 4k+7 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - mov d8, v6.d[1] // GHASH block 4k+2 - mid - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - rev w9, w12 // CTR block 4k+8 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - add w12, w12, #1 // CTR block 4k+8 - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - mov d6, v7.d[1] // GHASH block 4k+3 - mid - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - orr x9, x11, x9, lsl #32 // CTR block 4k+8 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - cmp x17, #12 // setup flags for AES-128/192/256 check - eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid - movi v8.8b, #0xc2 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - shl d8, d8, #56 // mod_constant - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - b.lt Ldec_main_loop_continue // branch if AES-128 - + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_prepretail // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 11 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - b.eq Ldec_main_loop_continue // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - -Ldec_main_loop_continue: - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - ldr q4, [x0, #0] // AES block 4k+4 - load ciphertext - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - ldr q5, [x0, #16] // AES block 4k+5 - load ciphertext - eor v0.16b, v4.16b, v0.16b // AES block 4k+4 - result - stp x23, x24, [x2], #16 // AES block 4k+3 - store result - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - ldr q7, [x0, #48] // AES block 4k+7 - load ciphertext - ldr q6, [x0, #32] // AES block 4k+6 - load ciphertext - mov x7, v0.d[1] // AES block 4k+4 - mov high - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - add x0, x0, #64 // AES input_ptr update - mov x6, v0.d[0] // AES block 4k+4 - mov low - fmov d0, x10 // CTR block 4k+8 - fmov v0.d[1], x9 // CTR block 4k+8 - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor v1.16b, v5.16b, v1.16b // AES block 4k+5 - result - rev w9, w12 // CTR block 4k+9 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - orr x9, x11, x9, lsl #32 // CTR block 4k+9 - cmp x0, x5 // LOOP CONTROL - add w12, w12, #1 // CTR block 4k+9 - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high - mov x20, v1.d[1] // AES block 4k+5 - mov high - eor v2.16b, v6.16b, v2.16b // AES block 4k+6 - result - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low - mov x19, v1.d[0] // AES block 4k+5 - mov low - fmov d1, x10 // CTR block 4k+9 - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - fmov v1.d[1], x9 // CTR block 4k+9 - rev w9, w12 // CTR block 4k+10 - add w12, w12, #1 // CTR block 4k+10 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - orr x9, x11, x9, lsl #32 // CTR block 4k+10 - rev64 v5.16b, v5.16b // GHASH block 4k+5 - eor x20, x20, x14 // AES block 4k+5 - round N high - stp x6, x7, [x2], #16 // AES block 4k+4 - store result - eor x19, x19, x13 // AES block 4k+5 - round N low - stp x19, x20, [x2], #16 // AES block 4k+5 - store result - rev64 v4.16b, v4.16b // GHASH block 4k+4 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - b.lt Ldec_main_loop - -Ldec_prepretail: // PREPRETAIL - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - mov x21, v2.d[0] // AES block 4k+2 - mov low - eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - mov x22, v2.d[1] // AES block 4k+2 - mov high - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d2, x10 // CTR block 4k+6 - fmov v2.d[1], x9 // CTR block 4k+6 - rev w9, w12 // CTR block 4k+7 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev64 v6.16b, v6.16b // GHASH block 4k+2 - orr x9, x11, x9, lsl #32 // CTR block 4k+7 - mov x23, v3.d[0] // AES block 4k+3 - mov low - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - mov x24, v3.d[1] // AES block 4k+3 - mov high - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - mov d8, v4.d[1] // GHASH block 4k - mid - fmov d3, x10 // CTR block 4k+7 - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - fmov v3.d[1], x9 // CTR block 4k+7 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - mov d10, v17.d[1] // GHASH block 4k - mid - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - rev64 v7.16b, v7.16b // GHASH block 4k+3 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - mov d8, v6.d[1] // GHASH block 4k+2 - mid - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - mov d6, v7.d[1] // GHASH block 4k+3 - mid - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - movi v8.8b, #0xc2 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low - pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - cmp x17, #12 // setup flags for AES-128/192/256 check - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - shl d8, d8, #56 // mod_constant - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - b.lt Ldec_finish_prepretail // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 11 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 11 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + aesmc v3.16b, v3.16b // AES block 3 - round 11 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 12 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 - b.eq Ldec_finish_prepretail // branch if AES-192 - - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 Ldec_finish_prepretail: - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor x22, x22, x14 // AES block 4k+2 - round N high - eor x23, x23, x13 // AES block 4k+3 - round N low - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - add w12, w12, #1 // CTR block 4k+7 - eor x21, x21, x13 // AES block 4k+2 - round N low - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor x24, x24, x14 // AES block 4k+3 - round N high - stp x21, x22, [x2], #16 // AES block 4k+2 - store result - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - stp x23, x24, [x2], #16 // AES block 4k+3 - store result - - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 Ldec_tail: // TAIL - sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process - ld1 { v5.16b}, [x0], #16 // AES block 4k+4 - load ciphertext - eor v0.16b, v5.16b, v0.16b // AES block 4k+4 - result - mov x6, v0.d[0] // AES block 4k+4 - mov low - mov x7, v0.d[1] // AES block 4k+4 - mov high - ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ld1 { v5.16b}, [x0], #16 // AES block 0 - load ciphertext + eor v0.16b, v5.16b, v0.16b // AES block 0 - result + mov x6, v0.d[0] // AES block 0 - mov low + mov x7, v0.d[1] // AES block 0 - mov high + ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high cmp x5, #48 - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high b.gt Ldec_blocks_more_than_3 - sub w12, w12, #1 mov v3.16b, v2.16b movi v10.8b, #0 movi v11.8b, #0 - cmp x5, #32 movi v9.8b, #0 mov v2.16b, v1.16b + cmp x5, #32 b.gt Ldec_blocks_more_than_2 - sub w12, w12, #1 mov v3.16b, v1.16b cmp x5, #16 b.gt Ldec_blocks_more_than_1 - sub w12, w12, #1 b Ldec_blocks_less_than_1 Ldec_blocks_more_than_3: // blocks left > 3 - rev64 v4.16b, v5.16b // GHASH final-3 block - ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext - stp x6, x7, [x2], #16 // AES final-3 block - store result - mov d10, v17.d[1] // GHASH final-3 block - mid - eor v4.16b, v4.16b, v8.16b // feed in partial tag - eor v0.16b, v5.16b, v1.16b // AES final-2 block - result - mov d22, v4.d[1] // GHASH final-3 block - mid - mov x6, v0.d[0] // AES final-2 block - mov low - mov x7, v0.d[1] // AES final-2 block - mov high - eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid - movi v8.8b, #0 // suppress further partial tag feed in - pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high - pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid - eor x6, x6, x13 // AES final-2 block - round N low - pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low - eor x7, x7, x14 // AES final-2 block - round N high + rev64 v4.16b, v5.16b // GHASH final-3 block + ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext + stp x6, x7, [x2], #16 // AES final-3 block - store result + mov d10, v17.d[1] // GHASH final-3 block - mid + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor v0.16b, v5.16b, v1.16b // AES final-2 block - result + mov d22, v4.d[1] // GHASH final-3 block - mid + mov x6, v0.d[0] // AES final-2 block - mov low + mov x7, v0.d[1] // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor x6, x6, x13 // AES final-2 block - round N low + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + eor x7, x7, x14 // AES final-2 block - round N high Ldec_blocks_more_than_2: // blocks left > 2 - rev64 v4.16b, v5.16b // GHASH final-2 block - ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext - eor v4.16b, v4.16b, v8.16b // feed in partial tag - stp x6, x7, [x2], #16 // AES final-2 block - store result - eor v0.16b, v5.16b, v2.16b // AES final-1 block - result - mov d22, v4.d[1] // GHASH final-2 block - mid - pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low - pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high - eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid - mov x6, v0.d[0] // AES final-1 block - mov low - mov x7, v0.d[1] // AES final-1 block - mov high - eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low - movi v8.8b, #0 // suppress further partial tag feed in - pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high - eor x6, x6, x13 // AES final-1 block - round N low - eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid - eor x7, x7, x14 // AES final-1 block - round N high + rev64 v4.16b, v5.16b // GHASH final-2 block + ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + stp x6, x7, [x2], #16 // AES final-2 block - store result + eor v0.16b, v5.16b, v2.16b // AES final-1 block - result + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + mov x6, v0.d[0] // AES final-1 block - mov low + mov x7, v0.d[1] // AES final-1 block - mov high + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + movi v8.8b, #0 // suppress further partial tag feed in + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + eor x6, x6, x13 // AES final-1 block - round N low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + eor x7, x7, x14 // AES final-1 block - round N high Ldec_blocks_more_than_1: // blocks left > 1 - stp x6, x7, [x2], #16 // AES final-1 block - store result - rev64 v4.16b, v5.16b // GHASH final-1 block - ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext - eor v4.16b, v4.16b, v8.16b // feed in partial tag - movi v8.8b, #0 // suppress further partial tag feed in - mov d22, v4.d[1] // GHASH final-1 block - mid - eor v0.16b, v5.16b, v3.16b // AES final block - result - pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high - eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid - pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low - mov x6, v0.d[0] // AES final block - mov low - ins v22.d[1], v22.d[0] // GHASH final-1 block - mid - mov x7, v0.d[1] // AES final block - mov high - pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid - eor x6, x6, x13 // AES final block - round N low - eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low - eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high - eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid - eor x7, x7, x14 // AES final block - round N high + stp x6, x7, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block + ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + movi v8.8b, #0 // suppress further partial tag feed in + mov d22, v4.d[1] // GHASH final-1 block - mid + eor v0.16b, v5.16b, v3.16b // AES final block - result + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + mov x6, v0.d[0] // AES final block - mov low + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + mov x7, v0.d[1] // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + eor x6, x6, x13 // AES final block - round N low + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor x7, x7, x14 // AES final block - round N high Ldec_blocks_less_than_1: // blocks left <= 1 - and x1, x1, #127 // bit_length %= 128 - mvn x14, xzr // rkN_h = 0xffffffffffffffff - sub x1, x1, #128 // bit_length -= 128 - mvn x13, xzr // rkN_l = 0xffffffffffffffff - ldp x4, x5, [x2] // load existing bytes we need to not overwrite - neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) - and x1, x1, #127 // bit_length %= 128 - lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x14, xzr // rkN_h = 0xffffffffffffffff + sub x1, x1, #128 // bit_length -= 128 + mvn x13, xzr // rkN_l = 0xffffffffffffffff + ldp x4, x5, [x2] // load existing bytes we need to not overwrite + neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block cmp x1, #64 csel x9, x13, x14, lt csel x10, x14, xzr, lt - fmov d0, x9 // ctr0b is mask for last block + fmov d0, x9 // ctr0b is mask for last block and x6, x6, x9 mov v0.d[1], x10 - bic x4, x4, x9 // mask out low existing bytes - rev w9, w12 - bic x5, x5, x10 // mask out high existing bytes + bic x4, x4, x9 // mask out low existing bytes + bic x5, x5, x10 // mask out high existing bytes orr x6, x6, x4 and x7, x7, x10 orr x7, x7, x5 - and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits - rev64 v4.16b, v5.16b // GHASH final block - eor v4.16b, v4.16b, v8.16b // feed in partial tag - pmull v21.1q, v4.1d, v12.1d // GHASH final block - low - mov d8, v4.d[1] // GHASH final block - mid - eor v8.8b, v8.8b, v4.8b // GHASH final block - mid - pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high - pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final block - high - eor v11.16b, v11.16b, v21.16b // GHASH final block - low - eor v10.16b, v10.16b, v8.16b // GHASH final block - mid - movi v8.8b, #0xc2 - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - shl d8, d8, #56 // mod_constant - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low + and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits + rev64 v4.16b, v5.16b // GHASH final block + eor v4.16b, v4.16b, v8.16b // feed in partial tag + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + mov d8, v4.d[1] // GHASH final block - mid + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + ldr d8, [sp, #128] + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b // MODULO - fold into low stp x6, x7, [x2] - str w9, [x16, #12] // store the updated counter - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + rev64 v11.16b, v11.16b // Final Tag mov x0, x15 - st1 { v11.16b }, [x3] + st1 { v11.16b }, [x3] // Store final tag ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x23, x24, [sp, #48] @@ -1547,9 +1470,1440 @@ ldp d10, d11, [sp, #80] ldp d12, d13, [sp, #96] ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 + ldp x29, x30, [sp], #224 AARCH64_VALIDATE_LINK_REGISTER ret -#endif + +.globl _aes_gcm_enc_kernel_eor3 +.private_extern _aes_gcm_enc_kernel_eor3 + +.align 4 +_aes_gcm_enc_kernel_eor3: + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + ld1 { v0.16b}, [x4] // Load initial counter block + stp x19, x20, [sp, #16] + mov v1.16b, v0.16b // Initialize ctr1-3 from ctr0 + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 // Pointer to counter block in memory + mov x8, x5 // Pointer to AES key schedule context + stp x21, x22, [sp, #32] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel_eor3 + stp d8, d9, [sp, #64] // Save Neon registers + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr w17, [x8, #240] // Load number of AES rounds + add x7, x8, x17, lsl #4 // Calculate pointer to the last round key + ldp x13, x14, [x7] // load round N key (for final XOR) + ldr q31, [x7, #-16] // load round N-1 key + add x4, x0, x1, lsr #3 // Calculate end of input + lsr x5, x1, #3 // Total byte length + mov x15, x5 + ldr w12, [x16, #12] // Load counter's low 32 bits + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes + add x5, x5, x0 + rev w12, w12 // Reverse for big-endian increment + uxtw x10, w12 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov x21, #0xc200000000000000 // GHASH reduction constant + str x21, [sp, #128] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., 160 + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str q0, [sp, #160] // Store base counter for block 0-3 + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev w20, w12 + mov v0.s[3], w20 // ctr0 + 0 + add w20, w12, #1 + rev w20, w20 + mov v1.s[3], w20 // ctr0 + 1 + add w20, w12, #2 + rev w20, w20 + mov v2.s[3], w20 // ctr0 + 2 + add w20, w12, #3 + rev w20, w20 + mov v3.s[3], w20 // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add w20, w12, #4 + rev w20, w20 + str w20, [sp, #172] // ctr0 + 4 for next iter + add w20, w12, #5 + rev w20, w20 + str w20, [sp, #188] // ctr0 + 5 for next iter + add w20, w12, #6 + rev w20, w20 + str w20, [sp, #204] // ctr0 + 6 for next iter + add w20, w12, #7 + rev w20, w20 + str w20, [sp, #220] // ctr0 + 7 for next iter + add w12, w12, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load H2, H3 (GHASH keys) + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 // Byte swap H3 for GHASH + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 // Byte swap H2 for GHASH + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + ldr q15, [x6, #80] // load H4 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 // Byte swap H4 for GHASH + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] // Load initial GHASH accumulator (T) + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap T for GHASH + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b // Correct byte order within 64-bit lanes + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // Karatsuba key: H4_low | H3_low + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load H1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 // Byte swap H1 for GHASH + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // Karatsuba key: H4_high | H3_high + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // Karatsuba key: H2_low | H1_low + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + ldr q30, [x7] // Preload round N key for final EOR + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_finish_first_blocks_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + b.eq Lenc_finish_first_blocks_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 +Lenc_finish_first_blocks_eor3: + cmp x0, x5 // check if we have <= 4 blocks to process in the tail + eor v17.16b, v17.16b, v9.16b // Karatsuba key: H3^H4 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // Karatsuba key: H2_high | H1_high + eor v16.16b, v16.16b, v8.16b // Karatsuba key: H1^H2 + b.ge Lenc_tail_eor3 // handle tail if no more full 4-block sets + ldp q6, q7, [x0, #32] // AES blocks 2,3 load plaintext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + eor3 v4.16b, v4.16b, v30.16b, v0.16b // AES block 0 - result = PT ^ AES(ctr0) + eor3 v5.16b, v5.16b, v30.16b, v1.16b // AES block 1 - result = PT ^ AES(ctr1) + eor3 v6.16b, v6.16b, v30.16b, v2.16b // AES block 2 - result = PT ^ AES(ctr2) + eor3 v7.16b, v7.16b, v30.16b, v3.16b // AES block 3 - result = PT ^ AES(ctr3) + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // Prepare and store counter values for the third iteration + rev w20, w12 + str w20, [sp, #172] // ctr + 8 + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] // ctr + 9 + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] // ctr + 10 + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] // ctr + 11 + add w12, w12, #4 // Advance counter base + cmp x0, x5 // check if we have <= 4 blocks remaining + b.ge Lenc_prepretail_eor3 // go to prepretail if < 2 full loops left +Lenc_main_loop_eor3: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + ldr d8, [sp, #128] // Load GHASH reduction constant + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 v4.16b, v4.16b // GHASH block 4k - Byte swap CT + rev64 v5.16b, v5.16b // GHASH block 4k+1 - Byte swap CT + rev64 v6.16b, v6.16b // GHASH block 4k+2 - Byte swap CT + rev64 v7.16b, v7.16b // GHASH block 4k+3 - Byte swap CT + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // GHASH - prepare acc for XOR + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid Karatsuba key + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d20, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v20.8b, v20.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + mov d21, v5.d[1] // GHASH block 4k+1 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + eor v21.8b, v21.8b, v5.8b // GHASH block 4k+1 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + pmull v10.1q, v20.1d, v10.1d // GHASH block 4k - mid + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + pmull v21.1q, v21.1d, v17.1d // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + eor v10.16b, v10.16b, v21.16b // GHASH block 4k+1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + ext v22.16b, v22.16b, v6.16b, #8 // GHASH block 4k+2 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v22.16b, v22.16b, v6.16b // GHASH block 4k+2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull2 v22.1q, v22.2d, v16.2d // GHASH block 4k+2 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + mov d23, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v23.8b, v23.8b, v7.8b // GHASH block 4k+3 - mid + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + pmull v23.1q, v23.1d, v16.1d // GHASH block 4k+3 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor3 v10.16b, v10.16b, v22.16b, v23.16b // GHASH block 4k+2/3 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v22.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v21.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v22.16b, v22.16b, v21.16b // GHASH block 4k+3 - high + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull2 v23.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + pmull v21.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull v20.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + eor3 v9.16b, v9.16b, v22.16b, v23.16b // GHASH block 4k/1/2/3 - high + pmull v22.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + ldp q6, q7, [x0, #32] + ldp q4, q5, [x0], #64 + eor v20.16b, v20.16b, v21.16b // GHASH block 4k+1 - low + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + eor3 v11.16b, v11.16b, v22.16b, v20.16b // GHASH block 4k/1/2/3 - low + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + eor3 v10.16b, v10.16b, v9.16b, v11.16b // MODULO - karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v20.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_main_loop_continue_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Lenc_main_loop_continue_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +Lenc_main_loop_continue_eor3: + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v20.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v20.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v9.16b, v11.16b, v20.16b // MODULO - fold into low + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor3 v4.16b, v4.16b, v30.16b, v0.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor3 v5.16b, v5.16b, v30.16b, v1.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor3 v6.16b, v6.16b, v30.16b, v2.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor3 v7.16b, v7.16b, v30.16b, v3.16b // AES block 4k+7 - result + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // We used these registers as temporaries above so reload the RKs. + ldp q20, q21, [x8, #32] // load rk2, rk3 + ldp q22, q23, [x8, #64] // load rk4, rk5 + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + rev w20, w12 + str w20, [sp, #172] + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] + add w12, w12, #4 + cmp x0, x5 // LOOP_eor3 CONTROL + b.lt Lenc_main_loop_eor3 +Lenc_prepretail_eor3: // PREPRETAIL + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + rev64 v6.16b, v6.16b // GHASH block 2 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + rev64 v4.16b, v4.16b // GHASH block 0 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v5.16b, v5.16b // GHASH block 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid Karatsuba key + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + mov d4, v5.d[1] // GHASH block 1 - mid + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + mov d8, v6.d[1] // GHASH block 2 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + ins v8.d[1], v8.d[0] // GHASH block 2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v9.16b, v9.16b, v4.16b // GHASH block 2 - high + mov d4, v7.d[1] // GHASH block 3 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + eor v4.8b, v4.8b, v7.8b // GHASH block 3 - mid + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull v4.1q, v4.1d, v16.1d // GHASH block 3 - mid + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + eor v9.16b, v9.16b, v5.16b // GHASH block 3 - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + ldr d8, [sp, #128] + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + eor v10.16b, v10.16b, v4.16b // GHASH block 3 - mid + pmull v6.1q, v7.1d, v12.1d // GHASH block 3 - low + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + eor v11.16b, v11.16b, v6.16b // GHASH block 3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v10.16b, v10.16b, v9.16b // karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + pmull v4.1q, v9.1d, v8.1d + ext v9.16b, v9.16b, v9.16b, #8 + eor v10.16b, v10.16b, v11.16b + b.lt Lenc_finish_prepretail_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Lenc_finish_prepretail_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Lenc_finish_prepretail_eor3: + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + eor3 v10.16b, v10.16b, v4.16b, v9.16b + pmull v4.1q, v10.1d, v8.1d + ext v10.16b, v10.16b, v10.16b, #8 + eor3 v11.16b, v11.16b, v4.16b, v10.16b +Lenc_tail_eor3: // TAIL: Process remaining 0 to 3 blocks + ext v8.16b, v11.16b, v11.16b, #8 // Save current GHASH state for partial tag feed-in + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ldp x6, x7, [x0], #16 // AES block 0 - load plaintext + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high + cmp x5, #48 + fmov d4, x6 // AES block 0 - mov low + fmov v4.d[1], x7 // AES block 0 - mov high + eor v5.16b, v4.16b, v0.16b // AES block 0 - result + b.gt Lenc_blocks_more_than_3_eor3 + cmp x5, #32 + mov v3.16b, v2.16b + movi v11.8b, #0 + movi v9.8b, #0 + mov v2.16b, v1.16b + movi v10.8b, #0 + b.gt Lenc_blocks_more_than_2_eor3 + mov v3.16b, v1.16b + cmp x5, #16 + b.gt Lenc_blocks_more_than_1_eor3 + b Lenc_blocks_less_than_1_eor3 +Lenc_blocks_more_than_3_eor3: // blocks left > 3 + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-3 block + eor x6, x6, x13 // AES final-2 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor x7, x7, x14 // AES final-2 block - round N high + mov d22, v4.d[1] // GHASH final-3 block - mid + fmov d5, x6 // AES final-2 block - mov low + fmov v5.d[1], x7 // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + mov d10, v17.d[1] // GHASH final-3 block - mid + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor v5.16b, v5.16b, v1.16b // AES final-2 block - result +Lenc_blocks_more_than_2_eor3: // blocks left > 2 + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-2 block + eor x6, x6, x13 // AES final-1 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + fmov d5, x6 // AES final-1 block - mov low + eor x7, x7, x14 // AES final-1 block - round N high + fmov v5.d[1], x7 // AES final-1 block - mov high + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + eor v5.16b, v5.16b, v2.16b // AES final-1 block - result + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid +Lenc_blocks_more_than_1_eor3: // blocks left > 1 + st1 { v5.16b}, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block: Byte Swap CT + ldp x6, x7, [x0], #16 // AES final block - load plaintext + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + movi v8.8b, #0 // Clear for next block + eor x6, x6, x13 // AES final block - round N low + mov d22, v4.d[1] // GHASH final-1 block - mid + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor x7, x7, x14 // AES final block - round N high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + fmov d5, x6 // AES final block - mov low + fmov v5.d[1], x7 // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + eor v5.16b, v5.16b, v3.16b // AES final block - result + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low +Lenc_blocks_less_than_1_eor3: // Last partial block handling + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x13, xzr // Mask for low 64 bits + sub x1, x1, #128 // + neg x1, x1 // Valid bits in the last block (1-128) + ldr q18, [x2] // Load destination for merging + mvn x14, xzr // Mask for high 64 bits + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + cmp x1, #64 + csel x6, x13, x14, lt + csel x7, x14, xzr, lt + fmov d0, x6 // ctr0d is mask for last block + fmov v0.d[1], x7 + and v5.16b, v5.16b, v0.16b // Mask out unused bits of the last CT block + rev64 v4.16b, v5.16b // GHASH final block - byte swap + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + bif v5.16b, v18.16b, v0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + mov d8, v4.d[1] // GHASH final block - mid + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + fmov d8, x21 + eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v7.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + st1 { v5.16b}, [x2] // store all 16B + eor3 v11.16b, v11.16b, v9.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap GHASH result + rev64 v11.16b, v11.16b // Final Tag + mov x0, x15 + st1 { v11.16b }, [x3] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret + +.globl _aes_gcm_dec_kernel_eor3 +.private_extern _aes_gcm_dec_kernel_eor3 + +.align 4 +_aes_gcm_dec_kernel_eor3: + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + stp x19, x20, [sp, #16] + ld1 { v0.16b}, [x4] + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 + mov x8, x5 + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp d8, d9, [sp, #64] + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr w17, [x8, #240] // Load number of AES rounds + add x19, x8, x17, lsl #4 // borrow input_l1 for last key + ldp x13, x14, [x19] // load round N keys + ldr q31, [x19, #-16] // load round N-1 keys + add x4, x0, x1, lsr #3 // end_input_ptr + lsr x5, x1, #3 // byte_len + mov x15, x5 + ldr w9, [x16, #12] // Load scalar 32-bit counter (CTR) + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + add x5, x5, x0 + rev w9, w9 // Reverse it once for big-endian incrementing + uxtw x10, w9 // Zero extend reversed w9 into x10 + str q0, [sp, #160] + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + rev w20, w9 + mov v0.s[3], w20 + add w20, w9, #1 + rev w20, w20 + mov v1.s[3], w20 + add w20, w9, #2 + rev w20, w20 + mov v2.s[3], w20 + add w20, w9, #3 + rev w20, w20 + mov v3.s[3], w20 + add w20, w9, #4 + rev w20, w20 + str w20, [sp, #172] + add w20, w9, #5 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #6 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #7 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov x21, #0xc200000000000000 + str x21, [sp, #128] + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load h2, h3 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 + ldr q15, [x6, #80] // load h4 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // h4l | h3l + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load h1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // h2l | h1l + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_first_blocks_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_first_blocks_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Ldec_finish_first_blocks_eor3: + ldr q27, [x19] // load rkN + cmp x0, x5 // check if we have <= 4 blocks + eor v17.16b, v17.16b, v9.16b // h4k | h3k + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // h2h | h1h + eor v16.16b, v16.16b, v8.16b // h2k | h1k + b.ge Ldec_tail_eor3 // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value (v30). This avoids several fmovs. + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + eor3 v0.16b, v4.16b, v0.16b, v27.16b // AES block 0 - result + eor3 v1.16b, v5.16b, v1.16b, v27.16b // AES block 1 - result + eor3 v2.16b, v6.16b, v2.16b, v27.16b // AES block 2 - result + eor3 v3.16b, v7.16b, v3.16b, v27.16b // AES block 3 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 0-3 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 // check if we have <= 4 blocks + b.ge Ldec_prepretail_eor3 // do prepretail +Ldec_main_loop_eor3: // main loop start + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + rev64 v4.16b, v4.16b // GHASH block 4k + rev64 v5.16b, v5.16b // GHASH block 4k+1 + rev64 v6.16b, v6.16b // GHASH block 4k+2 + rev64 v7.16b, v7.16b // GHASH block 4k+3 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // PRE 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d8, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + mov d4, v5.d[1] // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor3 v11.16b, v11.16b, v8.16b, v5.16b // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + mov d8, v6.d[1] // GHASH block 4k+2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid + ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v28.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + mov d6, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull v27.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + eor3 v10.16b, v10.16b, v4.16b, v8.16b // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid + eor3 v9.16b, v9.16b, v28.16b, v5.16b // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid + ldr d8, [sp, #128] + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + eor3 v10.16b, v10.16b, v6.16b, v7.16b // GHASH block 4k+3 - mid & MODULO - fold into mid + eor v11.16b, v11.16b, v27.16b // GHASH block 4k+3 - low + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v6.16b, v9.16b // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v11.16b, v8.16b, v10.16b // MODULO - fold into low + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_main_loop_continue_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Ldec_main_loop_continue_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +Ldec_main_loop_continue_eor3: + ldr q27, [x19] // load rkN + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor3 v0.16b, v4.16b, v0.16b, v27.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor3 v1.16b, v5.16b, v1.16b, v27.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor3 v2.16b, v6.16b, v2.16b, v27.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor3 v3.16b, v7.16b, v3.16b, v27.16b // AES block 4k+7 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 + b.lt Ldec_main_loop_eor3 +Ldec_prepretail_eor3: // PREPRETAIL + rev64 v4.16b, v4.16b // GHASH block 0 + rev64 v5.16b, v5.16b // GHASH block 1 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v6.16b, v6.16b // GHASH block 2 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + mov d4, v5.d[1] // GHASH block 1 - mid + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + mov d8, v6.d[1] // GHASH block 2 - mid + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + ins v8.d[1], v8.d[0] // GHASH block 2 - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + eor3 v9.16b, v9.16b, v4.16b, v5.16b // GHASH block 2 - high & GHASH block 3 - high + pmull v4.1q, v7.1d, v12.1d // GHASH block 3 - low + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + mov d6, v7.d[1] // GHASH block 3 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v6.8b, v6.8b, v7.8b // GHASH block 3 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + movi v8.8b, #0xc2 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v11.16b, v11.16b, v4.16b // GHASH block 3 - low + pmull v6.1q, v6.1d, v16.1d // GHASH block 3 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + eor v10.16b, v10.16b, v6.16b // GHASH block 3 - mid + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + shl d8, d8, #56 // mod_constant + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_prepretail_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_prepretail_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Ldec_finish_prepretail_eor3: + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v7.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v11.16b, v8.16b, v10.16b // MODULO - fold into low + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 +Ldec_tail_eor3: // TAIL + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ld1 { v5.16b}, [x0], #16 // AES block 0 - load ciphertext + eor v0.16b, v5.16b, v0.16b // AES block 0 - result + mov x6, v0.d[0] // AES block 0 - mov low + mov x7, v0.d[1] // AES block 0 - mov high + ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high + cmp x5, #48 + b.gt Ldec_blocks_more_than_3_eor3 + mov v3.16b, v2.16b + movi v10.8b, #0 + movi v11.8b, #0 + movi v9.8b, #0 + mov v2.16b, v1.16b + cmp x5, #32 + b.gt Ldec_blocks_more_than_2_eor3 + mov v3.16b, v1.16b + cmp x5, #16 + b.gt Ldec_blocks_more_than_1_eor3 + b Ldec_blocks_less_than_1_eor3 +Ldec_blocks_more_than_3_eor3: // blocks left > 3 + rev64 v4.16b, v5.16b // GHASH final-3 block + ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext + stp x6, x7, [x2], #16 // AES final-3 block - store result + mov d10, v17.d[1] // GHASH final-3 block - mid + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor v0.16b, v5.16b, v1.16b // AES final-2 block - result + mov d22, v4.d[1] // GHASH final-3 block - mid + mov x6, v0.d[0] // AES final-2 block - mov low + mov x7, v0.d[1] // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor x6, x6, x13 // AES final-2 block - round N low + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + eor x7, x7, x14 // AES final-2 block - round N high +Ldec_blocks_more_than_2_eor3: // blocks left > 2 + rev64 v4.16b, v5.16b // GHASH final-2 block + ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + stp x6, x7, [x2], #16 // AES final-2 block - store result + eor v0.16b, v5.16b, v2.16b // AES final-1 block - result + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + mov x6, v0.d[0] // AES final-1 block - mov low + mov x7, v0.d[1] // AES final-1 block - mov high + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + movi v8.8b, #0 // suppress further partial tag feed in + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + eor x6, x6, x13 // AES final-1 block - round N low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + eor x7, x7, x14 // AES final-1 block - round N high +Ldec_blocks_more_than_1_eor3: // blocks left > 1 + stp x6, x7, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block + ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + movi v8.8b, #0 // suppress further partial tag feed in + mov d22, v4.d[1] // GHASH final-1 block - mid + eor v0.16b, v5.16b, v3.16b // AES final block - result + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + mov x6, v0.d[0] // AES final block - mov low + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + mov x7, v0.d[1] // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + eor x6, x6, x13 // AES final block - round N low + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor x7, x7, x14 // AES final block - round N high +Ldec_blocks_less_than_1_eor3: // blocks left <= 1 + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x14, xzr // rkN_h = 0xffffffffffffffff + sub x1, x1, #128 // bit_length -= 128 + mvn x13, xzr // rkN_l = 0xffffffffffffffff + ldp x4, x5, [x2] // load existing bytes we need to not overwrite + neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + cmp x1, #64 + csel x9, x13, x14, lt + csel x10, x14, xzr, lt + fmov d0, x9 // ctr0b is mask for last block + and x6, x6, x9 + mov v0.d[1], x10 + bic x4, x4, x9 // mask out low existing bytes + bic x5, x5, x10 // mask out high existing bytes + orr x6, x6, x4 + and x7, x7, x10 + orr x7, x7, x5 + and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits + rev64 v4.16b, v5.16b // GHASH final block + eor v4.16b, v4.16b, v8.16b // feed in partial tag + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + mov d8, v4.d[1] // GHASH final block - mid + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + ldr d8, [sp, #128] + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b // MODULO - fold into low + stp x6, x7, [x2] + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 + rev64 v11.16b, v11.16b // Final Tag + mov x0, x15 + st1 { v11.16b }, [x3] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret + +#endif // __ARM_MAX_ARCH__ >= 8 #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__APPLE__)
diff --git a/gen/bcm/aesv8-gcm-armv8-linux.S b/gen/bcm/aesv8-gcm-armv8-linux.S index fc07f9a..9d99ec3 100644 --- a/gen/bcm/aesv8-gcm-armv8-linux.S +++ b/gen/bcm/aesv8-gcm-armv8-linux.S
@@ -5,8 +5,7 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__ELF__) #if __ARM_MAX_ARCH__ >= 8 - -.arch armv8-a+crypto +.arch armv8.2-a+crypto+sha3 .text .globl aes_gcm_enc_kernel .hidden aes_gcm_enc_kernel @@ -14,765 +13,731 @@ .align 4 aes_gcm_enc_kernel: AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! + stp x29, x30, [sp, #-224]! mov x29, sp + ld1 { v0.16b}, [x4] // .Load initial counter block stp x19, x20, [sp, #16] - mov x16, x4 - mov x8, x5 + mov v1.16b, v0.16b // Initialize ctr1-3 from ctr0 + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 // Pointer to counter block in memory + mov x8, x5 // Pointer to AES key schedule context stp x21, x22, [sp, #32] - stp x23, x24, [sp, #48] - stp d8, d9, [sp, #64] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel + stp d8, d9, [sp, #64] // Save Neon registers stp d10, d11, [sp, #80] stp d12, d13, [sp, #96] stp d14, d15, [sp, #112] - ldr w17, [x8, #240] - add x19, x8, x17, lsl #4 // borrow input_l1 for last key - ldp x13, x14, [x19] // load round N keys - ldr q31, [x19, #-16] // load round N-1 keys - add x4, x0, x1, lsr #3 // end_input_ptr - lsr x5, x1, #3 // byte_len + ldr w17, [x8, #240] // .Load number of AES rounds + add x7, x8, x17, lsl #4 // Calculate pointer to the last round key + ldp x13, x14, [x7] // load round N key (for final XOR) + ldr q31, [x7, #-16] // load round N-1 key + add x4, x0, x1, lsr #3 // Calculate end of input + lsr x5, x1, #3 // Total byte length mov x15, x5 - ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 - ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible - sub x5, x5, #1 // byte_len - 1 - ldr q18, [x8, #0] // load rk0 - and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - ldr q25, [x8, #112] // load rk7 + ldr w12, [x16, #12] // .Load counter's low 32 bits + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes add x5, x5, x0 - lsr x12, x11, #32 - fmov d2, x10 // CTR block 2 - orr w11, w11, w11 - rev w12, w12 // rev_ctr32 - fmov d1, x10 // CTR block 1 + rev w12, w12 // Reverse for big-endian increment + uxtw x10, w12 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov x21, #0xc200000000000000 // GHASH reduction constant + str x21, [sp, #128] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., 160 + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str q0, [sp, #160] // Store base counter for block 0-3 + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev w20, w12 + mov v0.s[3], w20 // ctr0 + 0 + add w20, w12, #1 + rev w20, w20 + mov v1.s[3], w20 // ctr0 + 1 + add w20, w12, #2 + rev w20, w20 + mov v2.s[3], w20 // ctr0 + 2 + add w20, w12, #3 + rev w20, w20 + mov v3.s[3], w20 // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add w20, w12, #4 + rev w20, w20 + str w20, [sp, #172] // ctr0 + 4 for next iter + add w20, w12, #5 + rev w20, w20 + str w20, [sp, #188] // ctr0 + 5 for next iter + add w20, w12, #6 + rev w20, w20 + str w20, [sp, #204] // ctr0 + 6 for next iter + add w20, w12, #7 + rev w20, w20 + str w20, [sp, #220] // ctr0 + 7 for next iter + add w12, w12, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 0 - round 0 - add w12, w12, #1 // increment rev_ctr32 - rev w9, w12 // CTR block 1 - fmov d3, x10 // CTR block 3 - orr x9, x11, x9, lsl #32 // CTR block 1 - add w12, w12, #1 // CTR block 1 - ldr q19, [x8, #16] // load rk1 - fmov v1.d[1], x9 // CTR block 1 - rev w9, w12 // CTR block 2 - add w12, w12, #1 // CTR block 2 - orr x9, x11, x9, lsl #32 // CTR block 2 - ldr q20, [x8, #32] // load rk2 - fmov v2.d[1], x9 // CTR block 2 - rev w9, w12 // CTR block 3 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 0 - round 1 - orr x9, x11, x9, lsl #32 // CTR block 3 - fmov v3.d[1], x9 // CTR block 3 + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 1 - round 0 - ldr q21, [x8, #48] // load rk3 - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 0 - round 2 - ldr q24, [x8, #96] // load rk6 + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 2 - round 0 - ldr q23, [x8, #80] // load rk5 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 1 - round 1 - ldr q14, [x6, #48] // load h3l | h3h - ext v14.16b, v14.16b, v14.16b, #8 + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 3 - round 0 + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load H2, H3 (GHASH keys) + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 2 - round 1 - ldr q22, [x8, #64] // load rk4 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 1 - round 2 - ldr q13, [x6, #32] // load h2l | h2h - ext v13.16b, v13.16b, v13.16b, #8 + aesmc v2.16b, v2.16b // AES block 2 - round 1 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 3 - round 1 - ldr q30, [x8, #192] // load rk12 + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 // Byte swap H3 for GHASH + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 // Byte swap H2 for GHASH aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 2 - round 2 - ldr q15, [x6, #80] // load h4l | h4h - ext v15.16b, v15.16b, v15.16b, #8 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 1 - round 3 - ldr q29, [x8, #176] // load rk11 + aesmc v2.16b, v2.16b // AES block 2 - round 2 + ldr q15, [x6, #80] // load H4 aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 3 - round 2 - ldr q26, [x8, #128] // load rk8 - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 2 - round 3 - add w12, w12, #1 // CTR block 3 + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 // Byte swap H4 for GHASH aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 0 - round 3 + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] // .Load initial GHASH accumulator (T) + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap T for GHASH + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b // Correct byte order within 64-bit lanes aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 3 - round 3 - ld1 { v11.16b}, [x3] - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 2 - round 4 + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // Karatsuba key: H4_low | H3_low aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 0 - round 4 + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load H1 aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 1 - round 4 + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 // Byte swap H1 for GHASH + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // Karatsuba key: H4_high | H3_high aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 3 - round 4 - cmp x17, #12 // setup flags for AES-128/192/256 check + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // Karatsuba key: H2_low | H1_low aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 0 - round 5 + aesmc v0.16b, v0.16b // AES block 0 - round 5 + ldr q30, [x7] // Preload round N key for final EOR aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 1 - round 5 + aesmc v1.16b, v1.16b // AES block 1 - round 5 aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 3 - round 5 + aesmc v3.16b, v3.16b // AES block 3 - round 5 aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 2 - round 5 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 1 - round 6 - trn2 v17.2d, v14.2d, v15.2d // h4l | h3l - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 3 - round 6 - ldr q27, [x8, #144] // load rk9 + aesmc v2.16b, v2.16b // AES block 2 - round 5 aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 0 - round 6 - ldr q12, [x6] // load h1l | h1h - ext v12.16b, v12.16b, v12.16b, #8 + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 2 - round 6 - ldr q28, [x8, #160] // load rk10 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 1 - round 7 - trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 0 - round 7 + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 2 - round 7 + aesmc v2.16b, v2.16b // AES block 2 - round 7 aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 3 - round 7 - trn2 v16.2d, v12.2d, v13.2d // h2l | h1l - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 1 - round 8 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 2 - round 8 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 3 - round 8 + aesmc v3.16b, v3.16b // AES block 3 - round 7 aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 0 - round 8 - b.lt .Lenc_finish_first_blocks // branch if AES-128 - + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt .Lenc_finish_first_blocks // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 1 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 2 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 9 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 3 - round 9 + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 0 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 1 - round 10 + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 2 - round 10 + aesmc v2.16b, v2.16b // AES block 2 - round 10 aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 3 - round 10 + aesmc v3.16b, v3.16b // AES block 3 - round 10 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 0 - round 10 - b.eq .Lenc_finish_first_blocks // branch if AES-192 - - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 1 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 2 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 0 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 3 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 1 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 2 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 0 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 3 - round 12 - + aesmc v0.16b, v0.16b // AES block 0 - round 10 + b.eq .Lenc_finish_first_blocks // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 .Lenc_finish_first_blocks: - cmp x0, x5 // check if we have <= 4 blocks - eor v17.16b, v17.16b, v9.16b // h4k | h3k - aese v2.16b, v31.16b // AES block 2 - round N-1 - trn1 v8.2d, v12.2d, v13.2d // h2h | h1h - aese v1.16b, v31.16b // AES block 1 - round N-1 - aese v0.16b, v31.16b // AES block 0 - round N-1 - aese v3.16b, v31.16b // AES block 3 - round N-1 - eor v16.16b, v16.16b, v8.16b // h2k | h1k - b.ge .Lenc_tail // handle tail - - ldp x19, x20, [x0, #16] // AES block 1 - load plaintext - rev w9, w12 // CTR block 4 - ldp x6, x7, [x0, #0] // AES block 0 - load plaintext - ldp x23, x24, [x0, #48] // AES block 3 - load plaintext - ldp x21, x22, [x0, #32] // AES block 2 - load plaintext - add x0, x0, #64 // AES input_ptr update - eor x19, x19, x13 // AES block 1 - round N low - eor x20, x20, x14 // AES block 1 - round N high - fmov d5, x19 // AES block 1 - mov low - eor x6, x6, x13 // AES block 0 - round N low - eor x7, x7, x14 // AES block 0 - round N high - eor x24, x24, x14 // AES block 3 - round N high - fmov d4, x6 // AES block 0 - mov low - cmp x0, x5 // check if we have <= 8 blocks - fmov v4.d[1], x7 // AES block 0 - mov high - eor x23, x23, x13 // AES block 3 - round N low - eor x21, x21, x13 // AES block 2 - round N low - fmov v5.d[1], x20 // AES block 1 - mov high - fmov d6, x21 // AES block 2 - mov low - add w12, w12, #1 // CTR block 4 - orr x9, x11, x9, lsl #32 // CTR block 4 - fmov d7, x23 // AES block 3 - mov low - eor x22, x22, x14 // AES block 2 - round N high - fmov v6.d[1], x22 // AES block 2 - mov high - eor v4.16b, v4.16b, v0.16b // AES block 0 - result - fmov d0, x10 // CTR block 4 - fmov v0.d[1], x9 // CTR block 4 - rev w9, w12 // CTR block 5 - add w12, w12, #1 // CTR block 5 - eor v5.16b, v5.16b, v1.16b // AES block 1 - result - fmov d1, x10 // CTR block 5 - orr x9, x11, x9, lsl #32 // CTR block 5 - fmov v1.d[1], x9 // CTR block 5 - rev w9, w12 // CTR block 6 - st1 { v4.16b}, [x2], #16 // AES block 0 - store result - fmov v7.d[1], x24 // AES block 3 - mov high - orr x9, x11, x9, lsl #32 // CTR block 6 - eor v6.16b, v6.16b, v2.16b // AES block 2 - result - st1 { v5.16b}, [x2], #16 // AES block 1 - store result - add w12, w12, #1 // CTR block 6 - fmov d2, x10 // CTR block 6 - fmov v2.d[1], x9 // CTR block 6 - st1 { v6.16b}, [x2], #16 // AES block 2 - store result - rev w9, w12 // CTR block 7 - orr x9, x11, x9, lsl #32 // CTR block 7 - eor v7.16b, v7.16b, v3.16b // AES block 3 - result - st1 { v7.16b}, [x2], #16 // AES block 3 - store result - b.ge .Lenc_prepretail // do prepretail - -.Lenc_main_loop: // main loop start + cmp x0, x5 // check if we have <= 4 blocks to process in the tail + eor v17.16b, v17.16b, v9.16b // Karatsuba key: H3^H4 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // Karatsuba key: H2_high | H1_high + eor v16.16b, v16.16b, v8.16b // Karatsuba key: H1^H2 + b.ge .Lenc_tail // handle tail if no more full 4-block sets + ldp q6, q7, [x0, #32] // AES blocks 2,3 load plaintext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + eor v4.16b, v4.16b, v30.16b + eor v4.16b, v4.16b, v0.16b // AES block 0 - result = PT ^ AES(ctr0) + eor v5.16b, v5.16b, v30.16b + eor v5.16b, v5.16b, v1.16b // AES block 1 - result = PT ^ AES(ctr1) + eor v6.16b, v6.16b, v30.16b + eor v6.16b, v6.16b, v2.16b // AES block 2 - result = PT ^ AES(ctr2) + eor v7.16b, v7.16b, v30.16b + eor v7.16b, v7.16b, v3.16b // AES block 3 - result = PT ^ AES(ctr3) + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // Prepare and store counter values for the third iteration + rev w20, w12 + str w20, [sp, #172] // ctr + 8 + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] // ctr + 9 + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] // ctr + 10 + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] // ctr + 11 + add w12, w12, #4 // Advance counter base + cmp x0, x5 // check if we have <= 4 blocks remaining + b.ge .Lenc_prepretail // go to prepretail if < 2 full loops left +.Lenc_main_loop: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d3, x10 // CTR block 4k+3 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - fmov v3.d[1], x9 // CTR block 4k+3 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - ldp x23, x24, [x0, #48] // AES block 4k+7 - load plaintext - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - ldp x21, x22, [x0, #32] // AES block 4k+6 - load plaintext - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - eor v4.16b, v4.16b, v11.16b // PRE 1 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - eor x23, x23, x13 // AES block 4k+7 - round N low - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - mov d10, v17.d[1] // GHASH block 4k - mid - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - eor x22, x22, x14 // AES block 4k+6 - round N high - mov d8, v4.d[1] // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + ldr d8, [sp, #128] // .Load GHASH reduction constant + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 v4.16b, v4.16b // GHASH block 4k - Byte swap CT + rev64 v5.16b, v5.16b // GHASH block 4k+1 - Byte swap CT + rev64 v6.16b, v6.16b // GHASH block 4k+2 - Byte swap CT + rev64 v7.16b, v7.16b // GHASH block 4k+3 - Byte swap CT + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // GHASH - prepare acc for XOR + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid Karatsuba key + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d20, v4.d[1] // GHASH block 4k - mid aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - mov d8, v6.d[1] // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v20.8b, v20.8b, v4.8b // GHASH block 4k - mid aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + mov d21, v5.d[1] // GHASH block 4k+1 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + eor v21.8b, v21.8b, v5.8b // GHASH block 4k+1 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + pmull v10.1q, v20.1d, v10.1d // GHASH block 4k - mid aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + pmull v21.1q, v21.1d, v17.1d // GHASH block 4k+1 - mid aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + eor v10.16b, v10.16b, v21.16b // GHASH block 4k+1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + ext v22.16b, v22.16b, v6.16b, #8 // GHASH block 4k+2 - mid aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v22.16b, v22.16b, v6.16b // GHASH block 4k+2 - mid aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull2 v22.1q, v22.2d, v16.2d // GHASH block 4k+2 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + mov d23, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v23.8b, v23.8b, v7.8b // GHASH block 4k+3 - mid aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - ldp x19, x20, [x0, #16] // AES block 4k+5 - load plaintext - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - mov d4, v7.d[1] // GHASH block 4k+3 - mid + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + pmull v23.1q, v23.1d, v16.1d // GHASH block 4k+3 - mid aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v10.16b, v10.16b, v22.16b + eor v10.16b, v10.16b, v23.16b // GHASH block 4k+2/3 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v22.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v21.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v22.16b, v22.16b, v21.16b // GHASH block 4k+3 - high + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull2 v23.1q, v5.2d, v14.2d // GHASH block 4k+1 - high aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor x19, x19, x13 // AES block 4k+5 - round N low - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + pmull v21.1q, v6.1d, v13.1d // GHASH block 4k+2 - low aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - eor x21, x21, x13 // AES block 4k+6 - round N low + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull v20.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + eor v9.16b, v9.16b, v22.16b + eor v9.16b, v9.16b, v23.16b // GHASH block 4k/1/2/3 - high + pmull v22.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + ldp q6, q7, [x0, #32] + ldp q4, q5, [x0], #64 + eor v20.16b, v20.16b, v21.16b // GHASH block 4k+1 - low + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + eor v11.16b, v11.16b, v22.16b + eor v11.16b, v11.16b, v20.16b // GHASH block 4k/1/2/3 - low + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + eor v10.16b, v10.16b, v9.16b + eor v10.16b, v10.16b, v11.16b // MODULO - karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v20.1q, v9.1d, v8.1d // MODULO - top 64b align with mid aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - movi v8.8b, #0xc2 - pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - cmp x17, #12 // setup flags for AES-128/192/256 check - fmov d5, x19 // AES block 4k+5 - mov low - ldp x6, x7, [x0, #0] // AES block 4k+4 - load plaintext - b.lt .Lenc_main_loop_continue // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt .Lenc_main_loop_continue // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - b.eq .Lenc_main_loop_continue // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq .Lenc_main_loop_continue // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 .Lenc_main_loop_continue: - shl d8, d8, #56 // mod_constant - eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid - add w12, w12, #1 // CTR block 4k+3 - eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - add x0, x0, #64 // AES input_ptr update - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - rev w9, w12 // CTR block 4k+8 - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor x6, x6, x13 // AES block 4k+4 - round N low - eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up - eor x7, x7, x14 // AES block 4k+4 - round N high - fmov d4, x6 // AES block 4k+4 - mov low - orr x9, x11, x9, lsl #32 // CTR block 4k+8 - eor v7.16b, v9.16b, v7.16b // MODULO - fold into mid - eor x20, x20, x14 // AES block 4k+5 - round N high - eor x24, x24, x14 // AES block 4k+7 - round N high - add w12, w12, #1 // CTR block 4k+8 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - fmov v4.d[1], x7 // AES block 4k+4 - mov high - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - fmov d7, x23 // AES block 4k+7 - mov low - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - fmov v5.d[1], x20 // AES block 4k+5 - mov high - fmov d6, x21 // AES block 4k+6 - mov low - cmp x0, x5 // .LOOP CONTROL - fmov v6.d[1], x22 // AES block 4k+6 - mov high - pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result - fmov d0, x10 // CTR block 4k+8 - fmov v0.d[1], x9 // CTR block 4k+8 - rev w9, w12 // CTR block 4k+9 - add w12, w12, #1 // CTR block 4k+9 - eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result - fmov d1, x10 // CTR block 4k+9 - orr x9, x11, x9, lsl #32 // CTR block 4k+9 - fmov v1.d[1], x9 // CTR block 4k+9 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - rev w9, w12 // CTR block 4k+10 - st1 { v4.16b}, [x2], #16 // AES block 4k+4 - store result - orr x9, x11, x9, lsl #32 // CTR block 4k+10 - eor v11.16b, v11.16b, v9.16b // MODULO - fold into low - fmov v7.d[1], x24 // AES block 4k+7 - mov high - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - st1 { v5.16b}, [x2], #16 // AES block 4k+5 - store result - add w12, w12, #1 // CTR block 4k+10 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result - fmov d2, x10 // CTR block 4k+10 - st1 { v6.16b}, [x2], #16 // AES block 4k+6 - store result - fmov v2.d[1], x9 // CTR block 4k+10 - rev w9, w12 // CTR block 4k+11 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - orr x9, x11, x9, lsl #32 // CTR block 4k+11 - eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result - st1 { v7.16b}, [x2], #16 // AES block 4k+7 - store result + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v20.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v20.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v9.16b, v11.16b + eor v11.16b, v11.16b, v20.16b // MODULO - fold into low + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor v4.16b, v4.16b, v30.16b + eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor v5.16b, v5.16b, v30.16b + eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor v6.16b, v6.16b, v30.16b + eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor v7.16b, v7.16b, v30.16b + eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // We used these registers as temporaries above so reload the RKs. + ldp q20, q21, [x8, #32] // load rk2, rk3 + ldp q22, q23, [x8, #64] // load rk4, rk5 + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + rev w20, w12 + str w20, [sp, #172] + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] + add w12, w12, #4 + cmp x0, x5 // .LOOP CONTROL b.lt .Lenc_main_loop - .Lenc_prepretail: // PREPRETAIL aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) + aesmc v1.16b, v1.16b // AES block 1 - round 0 + rev64 v6.16b, v6.16b // GHASH block 2 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - fmov d3, x10 // CTR block 4k+3 + aesmc v2.16b, v2.16b // AES block 2 - round 0 aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) - fmov v3.d[1], x9 // CTR block 4k+3 - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aesmc v0.16b, v0.16b // AES block 0 - round 0 + rev64 v4.16b, v4.16b // GHASH block 0 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aesmc v2.16b, v2.16b // AES block 2 - round 1 aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v5.16b, v5.16b // GHASH block 1 aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + aesmc v2.16b, v2.16b // AES block 2 - round 2 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - mov d10, v17.d[1] // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 3 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid Karatsuba key aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - mov d8, v4.d[1] // GHASH block 4k - mid - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + aesmc v2.16b, v2.16b // AES block 2 - round 3 aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + aesmc v0.16b, v0.16b // AES block 0 - round 2 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + aesmc v3.16b, v3.16b // AES block 3 - round 1 aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aesmc v1.16b, v1.16b // AES block 1 - round 3 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - mov d4, v5.d[1] // GHASH block 4k+1 - mid + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + mov d4, v5.d[1] // GHASH block 1 - mid aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low + aesmc v0.16b, v0.16b // AES block 0 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - mov d8, v6.d[1] // GHASH block 4k+2 - mid + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + mov d8, v6.d[1] // GHASH block 2 - mid aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) + aesmc v0.16b, v0.16b // AES block 0 - round 4 + rev64 v7.16b, v7.16b // GHASH block 3 aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - add w12, w12, #1 // CTR block 4k+3 - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aesmc v3.16b, v3.16b // AES block 3 - round 5 aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 2 - round 4 + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + ins v8.d[1], v8.d[0] // GHASH block 2 - mid aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - mov d4, v7.d[1] // GHASH block 4k+3 - mid + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v9.16b, v9.16b, v4.16b // GHASH block 2 - high + mov d4, v7.d[1] // GHASH block 3 - mid aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + eor v4.8b, v4.8b, v7.8b // GHASH block 3 - mid + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid + aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull v4.1q, v4.1d, v16.1d // GHASH block 3 - mid + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + aesmc v0.16b, v0.16b // AES block 0 - round 5 aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + aesmc v1.16b, v1.16b // AES block 1 - round 6 aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + aesmc v2.16b, v2.16b // AES block 2 - round 6 aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - movi v8.8b, #0xc2 + aesmc v0.16b, v0.16b // AES block 0 - round 6 aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + aesmc v3.16b, v3.16b // AES block 3 - round 6 aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high + aesmc v1.16b, v1.16b // AES block 1 - round 7 + eor v9.16b, v9.16b, v5.16b // GHASH block 3 - high aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + aesmc v0.16b, v0.16b // AES block 0 - round 7 aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - shl d8, d8, #56 // mod_constant + aesmc v3.16b, v3.16b // AES block 3 - round 7 + ldr d8, [sp, #128] aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid - pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aesmc v1.16b, v1.16b // AES block 1 - round 8 + eor v10.16b, v10.16b, v4.16b // GHASH block 3 - mid + pmull v6.1q, v7.1d, v12.1d // GHASH block 3 - low aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - cmp x17, #12 // setup flags for AES-128/192/256 check + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low + aesmc v0.16b, v0.16b // AES block 0 - round 8 + eor v11.16b, v11.16b, v6.16b // GHASH block 3 - low aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v10.16b, v10.16b, v9.16b // karatsuba tidy up + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v10.16b, v10.16b, v9.16b // karatsuba tidy up aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + aesmc v2.16b, v2.16b // AES block 2 - round 8 pmull v4.1q, v9.1d, v8.1d ext v9.16b, v9.16b, v9.16b, #8 eor v10.16b, v10.16b, v11.16b - b.lt .Lenc_finish_prepretail // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + b.lt .Lenc_finish_prepretail // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - b.eq .Lenc_finish_prepretail // branch if AES-192 - - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq .Lenc_finish_prepretail // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 .Lenc_finish_prepretail: + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 eor v10.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v9.16b pmull v4.1q, v10.1d, v8.1d ext v10.16b, v10.16b, v10.16b, #8 - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 eor v11.16b, v11.16b, v4.16b - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 eor v11.16b, v11.16b, v10.16b - -.Lenc_tail: // TAIL - ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag - sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process - ldp x6, x7, [x0], #16 // AES block 4k+4 - load plaintext - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high +.Lenc_tail: // TAIL: Process remaining 0 to 3 blocks + ext v8.16b, v11.16b, v11.16b, #8 // Save current GHASH state for partial tag feed-in + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ldp x6, x7, [x0], #16 // AES block 0 - load plaintext + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high cmp x5, #48 - fmov d4, x6 // AES block 4k+4 - mov low - fmov v4.d[1], x7 // AES block 4k+4 - mov high - eor v5.16b, v4.16b, v0.16b // AES block 4k+4 - result + fmov d4, x6 // AES block 0 - mov low + fmov v4.d[1], x7 // AES block 0 - mov high + eor v5.16b, v4.16b, v0.16b // AES block 0 - result b.gt .Lenc_blocks_more_than_3 cmp x5, #32 mov v3.16b, v2.16b movi v11.8b, #0 movi v9.8b, #0 - sub w12, w12, #1 mov v2.16b, v1.16b movi v10.8b, #0 b.gt .Lenc_blocks_more_than_2 mov v3.16b, v1.16b - sub w12, w12, #1 cmp x5, #16 b.gt .Lenc_blocks_more_than_1 - sub w12, w12, #1 b .Lenc_blocks_less_than_1 .Lenc_blocks_more_than_3: // blocks left > 3 - st1 { v5.16b}, [x2], #16 // AES final-3 block - store result - ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high - rev64 v4.16b, v5.16b // GHASH final-3 block - eor x6, x6, x13 // AES final-2 block - round N low - eor v4.16b, v4.16b, v8.16b // feed in partial tag - eor x7, x7, x14 // AES final-2 block - round N high - mov d22, v4.d[1] // GHASH final-3 block - mid - fmov d5, x6 // AES final-2 block - mov low - fmov v5.d[1], x7 // AES final-2 block - mov high - eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid - movi v8.8b, #0 // suppress further partial tag feed in - mov d10, v17.d[1] // GHASH final-3 block - mid - pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low - pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high - pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid - eor v5.16b, v5.16b, v1.16b // AES final-2 block - result + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-3 block + eor x6, x6, x13 // AES final-2 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor x7, x7, x14 // AES final-2 block - round N high + mov d22, v4.d[1] // GHASH final-3 block - mid + fmov d5, x6 // AES final-2 block - mov low + fmov v5.d[1], x7 // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + mov d10, v17.d[1] // GHASH final-3 block - mid + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor v5.16b, v5.16b, v1.16b // AES final-2 block - result .Lenc_blocks_more_than_2: // blocks left > 2 - st1 { v5.16b}, [x2], #16 // AES final-2 block - store result - ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high - rev64 v4.16b, v5.16b // GHASH final-2 block - eor x6, x6, x13 // AES final-1 block - round N low - eor v4.16b, v4.16b, v8.16b // feed in partial tag - fmov d5, x6 // AES final-1 block - mov low - eor x7, x7, x14 // AES final-1 block - round N high - fmov v5.d[1], x7 // AES final-1 block - mov high - movi v8.8b, #0 // suppress further partial tag feed in - pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high - mov d22, v4.d[1] // GHASH final-2 block - mid - pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low - eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid - eor v5.16b, v5.16b, v2.16b // AES final-1 block - result - eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high - pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low - eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-2 block + eor x6, x6, x13 // AES final-1 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + fmov d5, x6 // AES final-1 block - mov low + eor x7, x7, x14 // AES final-1 block - round N high + fmov v5.d[1], x7 // AES final-1 block - mov high + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + eor v5.16b, v5.16b, v2.16b // AES final-1 block - result + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid .Lenc_blocks_more_than_1: // blocks left > 1 - st1 { v5.16b}, [x2], #16 // AES final-1 block - store result - rev64 v4.16b, v5.16b // GHASH final-1 block - ldp x6, x7, [x0], #16 // AES final block - load input low & high - eor v4.16b, v4.16b, v8.16b // feed in partial tag - movi v8.8b, #0 // suppress further partial tag feed in - eor x6, x6, x13 // AES final block - round N low - mov d22, v4.d[1] // GHASH final-1 block - mid - pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high - eor x7, x7, x14 // AES final block - round N high - eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high - ins v22.d[1], v22.d[0] // GHASH final-1 block - mid - fmov d5, x6 // AES final block - mov low - fmov v5.d[1], x7 // AES final block - mov high - pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid - pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low - eor v5.16b, v5.16b, v3.16b // AES final block - result - eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low -.Lenc_blocks_less_than_1: // blocks left <= 1 - and x1, x1, #127 // bit_length %= 128 - mvn x13, xzr // rkN_l = 0xffffffffffffffff - sub x1, x1, #128 // bit_length -= 128 - neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) - ld1 { v18.16b}, [x2] // load existing bytes where the possibly partial last block is to be stored - mvn x14, xzr // rkN_h = 0xffffffffffffffff - and x1, x1, #127 // bit_length %= 128 - lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + st1 { v5.16b}, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block: Byte Swap CT + ldp x6, x7, [x0], #16 // AES final block - load plaintext + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + movi v8.8b, #0 // Clear for next block + eor x6, x6, x13 // AES final block - round N low + mov d22, v4.d[1] // GHASH final-1 block - mid + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor x7, x7, x14 // AES final block - round N high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + fmov d5, x6 // AES final block - mov low + fmov v5.d[1], x7 // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + eor v5.16b, v5.16b, v3.16b // AES final block - result + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low +.Lenc_blocks_less_than_1: // .Last partial block handling + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x13, xzr // Mask for low 64 bits + sub x1, x1, #128 // + neg x1, x1 // Valid bits in the last block (1-128) + ldr q18, [x2] // .Load destination for merging + mvn x14, xzr // Mask for high 64 bits + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block cmp x1, #64 csel x6, x13, x14, lt csel x7, x14, xzr, lt - fmov d0, x6 // ctr0b is mask for last block + fmov d0, x6 // ctr0d is mask for last block fmov v0.d[1], x7 - and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits - rev64 v4.16b, v5.16b // GHASH final block - eor v4.16b, v4.16b, v8.16b // feed in partial tag - bif v5.16b, v18.16b, v0.16b // insert existing bytes in top end of result before storing - pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high - mov d8, v4.d[1] // GHASH final block - mid - rev w9, w12 - pmull v21.1q, v4.1d, v12.1d // GHASH final block - low - eor v9.16b, v9.16b, v20.16b // GHASH final block - high - eor v8.8b, v8.8b, v4.8b // GHASH final block - mid - pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final block - low - eor v10.16b, v10.16b, v8.16b // GHASH final block - mid - movi v8.8b, #0xc2 - eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - shl d8, d8, #56 // mod_constant - eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - str w9, [x16, #12] // store the updated counter - st1 { v5.16b}, [x2] // store all 16B - eor v11.16b, v11.16b, v9.16b // MODULO - fold into low - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + and v5.16b, v5.16b, v0.16b // Mask out unused bits of the last CT block + rev64 v4.16b, v5.16b // GHASH final block - byte swap + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + bif v5.16b, v18.16b, v0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + mov d8, v4.d[1] // GHASH final block - mid + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + fmov d8, x21 + eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + st1 { v5.16b}, [x2] // store all 16B + eor v11.16b, v11.16b, v9.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap GHASH result + rev64 v11.16b, v11.16b // Final Tag mov x0, x15 - st1 { v11.16b }, [x3] + st1 { v11.16b }, [x3] // Store final tag ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] ldp d8, d9, [sp, #64] ldp d10, d11, [sp, #80] ldp d12, d13, [sp, #96] ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 + ldp x29, x30, [sp], #224 AARCH64_VALIDATE_LINK_REGISTER ret .size aes_gcm_enc_kernel,.-aes_gcm_enc_kernel @@ -782,9 +747,13 @@ .align 4 aes_gcm_dec_kernel: AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! + stp x29, x30, [sp, #-224]! mov x29, sp stp x19, x20, [sp, #16] + ld1 { v0.16b}, [x4] + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b mov x16, x4 mov x8, x5 stp x21, x22, [sp, #32] @@ -793,752 +762,705 @@ stp d10, d11, [sp, #80] stp d12, d13, [sp, #96] stp d14, d15, [sp, #112] - ldr w17, [x8, #240] - add x19, x8, x17, lsl #4 // borrow input_l1 for last key - ldp x13, x14, [x19] // load round N keys - ldr q31, [x19, #-16] // load round N-1 keys - lsr x5, x1, #3 // byte_len + ldr w17, [x8, #240] // .Load number of AES rounds + add x19, x8, x17, lsl #4 // borrow input_l1 for last key + ldp x13, x14, [x19] // load round N keys + ldr q31, [x19, #-16] // load round N-1 keys + add x4, x0, x1, lsr #3 // end_input_ptr + lsr x5, x1, #3 // byte_len mov x15, x5 - ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 - ldr q26, [x8, #128] // load rk8 - sub x5, x5, #1 // byte_len - 1 - ldr q25, [x8, #112] // load rk7 - and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - add x4, x0, x1, lsr #3 // end_input_ptr - ldr q24, [x8, #96] // load rk6 - lsr x12, x11, #32 - ldr q23, [x8, #80] // load rk5 - orr w11, w11, w11 - ldr q21, [x8, #48] // load rk3 + ldr w9, [x16, #12] // .Load scalar 32-bit counter (CTR) + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) add x5, x5, x0 - rev w12, w12 // rev_ctr32 - add w12, w12, #1 // increment rev_ctr32 - fmov d3, x10 // CTR block 3 - rev w9, w12 // CTR block 1 - add w12, w12, #1 // CTR block 1 - fmov d1, x10 // CTR block 1 - orr x9, x11, x9, lsl #32 // CTR block 1 - ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible - fmov v1.d[1], x9 // CTR block 1 - rev w9, w12 // CTR block 2 - add w12, w12, #1 // CTR block 2 - fmov d2, x10 // CTR block 2 - orr x9, x11, x9, lsl #32 // CTR block 2 - fmov v2.d[1], x9 // CTR block 2 - rev w9, w12 // CTR block 3 - orr x9, x11, x9, lsl #32 // CTR block 3 - ldr q18, [x8, #0] // load rk0 - fmov v3.d[1], x9 // CTR block 3 - add w12, w12, #1 // CTR block 3 - ldr q22, [x8, #64] // load rk4 - ldr q19, [x8, #16] // load rk1 + rev w9, w9 // Reverse it once for big-endian incrementing + uxtw x10, w9 // Zero extend reversed w9 into x10 + str q0, [sp, #160] + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + rev w20, w9 + mov v0.s[3], w20 + add w20, w9, #1 + rev w20, w20 + mov v1.s[3], w20 + add w20, w9, #2 + rev w20, w20 + mov v2.s[3], w20 + add w20, w9, #3 + rev w20, w20 + mov v3.s[3], w20 + add w20, w9, #4 + rev w20, w20 + str w20, [sp, #172] + add w20, w9, #5 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #6 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #7 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov x21, #0xc200000000000000 + str x21, [sp, #128] + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load h2, h3 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 + ldr q15, [x6, #80] // load h4 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // h4l | h3l + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load h1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // h2l | h1l + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_finish_first_blocks // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq .Ldec_finish_first_blocks // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +.Ldec_finish_first_blocks: + ldr q27, [x19] // load rkN + cmp x0, x5 // check if we have <= 4 blocks + eor v17.16b, v17.16b, v9.16b // h4k | h3k + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // h2h | h1h + eor v16.16b, v16.16b, v8.16b // h2k | h1k + b.ge .Ldec_tail // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value (v30). This avoids several fmovs. + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + eor v0.16b, v4.16b, v0.16b + eor v0.16b, v0.16b, v27.16b // AES block 0 - result + eor v1.16b, v5.16b, v1.16b + eor v1.16b, v1.16b, v27.16b // AES block 1 - result + eor v2.16b, v6.16b, v2.16b + eor v2.16b, v2.16b, v27.16b // AES block 2 - result + eor v3.16b, v7.16b, v3.16b + eor v3.16b, v3.16b, v27.16b // AES block 3 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 0-3 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 // check if we have <= 4 blocks + b.ge .Ldec_prepretail // do prepretail +.Ldec_main_loop: // main loop start + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + rev64 v4.16b, v4.16b // GHASH block 4k + rev64 v5.16b, v5.16b // GHASH block 4k+1 + rev64 v6.16b, v6.16b // GHASH block 4k+2 + rev64 v7.16b, v7.16b // GHASH block 4k+3 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // PRE 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d8, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + mov d4, v5.d[1] // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v5.16b // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + mov d8, v6.d[1] // GHASH block 4k+2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid + ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v28.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + mov d6, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull v27.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + eor v10.16b, v10.16b, v4.16b + eor v10.16b, v10.16b, v8.16b // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid + eor v9.16b, v9.16b, v28.16b + eor v9.16b, v9.16b, v5.16b // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid + ldr d8, [sp, #128] + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + eor v10.16b, v10.16b, v6.16b + eor v10.16b, v10.16b, v7.16b // GHASH block 4k+3 - mid & MODULO - fold into mid + eor v11.16b, v11.16b, v27.16b // GHASH block 4k+3 - low + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v6.16b + eor v10.16b, v10.16b, v9.16b // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_main_loop_continue // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq .Ldec_main_loop_continue // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +.Ldec_main_loop_continue: + ldr q27, [x19] // load rkN + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor v0.16b, v4.16b, v0.16b + eor v0.16b, v0.16b, v27.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor v1.16b, v5.16b, v1.16b + eor v1.16b, v1.16b, v27.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor v2.16b, v6.16b, v2.16b + eor v2.16b, v2.16b, v27.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor v3.16b, v7.16b, v3.16b + eor v3.16b, v3.16b, v27.16b // AES block 4k+7 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 + b.lt .Ldec_main_loop +.Ldec_prepretail: // PREPRETAIL + rev64 v4.16b, v4.16b // GHASH block 0 + rev64 v5.16b, v5.16b // GHASH block 1 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 aese v0.16b, v18.16b aesmc v0.16b, v0.16b // AES block 0 - round 0 - ldr q14, [x6, #48] // load h3l | h3h - ext v14.16b, v14.16b, v14.16b, #8 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 3 - round 0 - ldr q15, [x6, #80] // load h4l | h4h - ext v15.16b, v15.16b, v15.16b, #8 aese v1.16b, v18.16b aesmc v1.16b, v1.16b // AES block 1 - round 0 - ldr q13, [x6, #32] // load h2l | h2h - ext v13.16b, v13.16b, v13.16b, #8 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 2 - round 0 - ldr q20, [x8, #32] // load rk2 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v6.16b, v6.16b // GHASH block 2 aese v1.16b, v19.16b aesmc v1.16b, v1.16b // AES block 1 - round 1 - ld1 { v11.16b}, [x3] - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high aese v2.16b, v19.16b aesmc v2.16b, v2.16b // AES block 2 - round 1 - ldr q27, [x8, #144] // load rk9 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low aese v3.16b, v19.16b aesmc v3.16b, v3.16b // AES block 3 - round 1 - ldr q30, [x8, #192] // load rk12 + mov d4, v5.d[1] // GHASH block 1 - mid aese v0.16b, v20.16b aesmc v0.16b, v0.16b // AES block 0 - round 2 - ldr q12, [x6] // load h1l | h1h - ext v12.16b, v12.16b, v12.16b, #8 - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 2 - round 2 - ldr q28, [x8, #160] // load rk10 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 3 - round 2 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 0 - round 3 aese v1.16b, v20.16b aesmc v1.16b, v1.16b // AES block 1 - round 2 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + mov d8, v6.d[1] // GHASH block 2 - mid + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low aese v0.16b, v22.16b aesmc v0.16b, v0.16b // AES block 0 - round 4 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + ins v8.d[1], v8.d[0] // GHASH block 2 - mid aese v2.16b, v21.16b aesmc v2.16b, v2.16b // AES block 2 - round 3 aese v1.16b, v21.16b aesmc v1.16b, v1.16b // AES block 1 - round 3 - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 3 - round 4 + eor v9.16b, v9.16b, v4.16b + eor v9.16b, v9.16b, v5.16b // GHASH block 2 - high & GHASH block 3 - high + pmull v4.1q, v7.1d, v12.1d // GHASH block 3 - low aese v2.16b, v22.16b aesmc v2.16b, v2.16b // AES block 2 - round 4 + mov d6, v7.d[1] // GHASH block 3 - mid aese v1.16b, v22.16b aesmc v1.16b, v1.16b // AES block 1 - round 4 - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 3 - round 5 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 0 - round 5 - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid aese v2.16b, v23.16b aesmc v2.16b, v2.16b // AES block 2 - round 5 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 0 - round 6 + eor v6.8b, v6.8b, v7.8b // GHASH block 3 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 aese v3.16b, v24.16b aesmc v3.16b, v3.16b // AES block 3 - round 6 - cmp x17, #12 // setup flags for AES-128/192/256 check - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid aese v2.16b, v24.16b aesmc v2.16b, v2.16b // AES block 2 - round 6 - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 0 - round 7 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + movi v8.8b, #0xc2 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v11.16b, v11.16b, v4.16b // GHASH block 3 - low + pmull v6.1q, v6.1d, v16.1d // GHASH block 3 - mid aese v3.16b, v25.16b aesmc v3.16b, v3.16b // AES block 3 - round 7 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 0 - round 8 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + eor v10.16b, v10.16b, v6.16b // GHASH block 3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b // AES block 3 - round 8 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up aese v1.16b, v26.16b aesmc v1.16b, v1.16b // AES block 1 - round 8 - ldr q29, [x8, #176] // load rk11 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + shl d8, d8, #56 // mod_constant aese v2.16b, v26.16b aesmc v2.16b, v2.16b // AES block 2 - round 8 - b.lt .Ldec_finish_first_blocks // branch if AES-128 - + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_finish_prepretail // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 0 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 1 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 3 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 2 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 0 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 10 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 1 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 3 - round 10 + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 2 - round 10 - b.eq .Ldec_finish_first_blocks // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 0 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 3 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 1 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 2 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 1 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 0 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 2 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 3 - round 12 - -.Ldec_finish_first_blocks: - cmp x0, x5 // check if we have <= 4 blocks - trn1 v9.2d, v14.2d, v15.2d // h4h | h3h - trn2 v17.2d, v14.2d, v15.2d // h4l | h3l - trn1 v8.2d, v12.2d, v13.2d // h2h | h1h - trn2 v16.2d, v12.2d, v13.2d // h2l | h1l - eor v17.16b, v17.16b, v9.16b // h4k | h3k - aese v1.16b, v31.16b // AES block 1 - round N-1 - aese v2.16b, v31.16b // AES block 2 - round N-1 - eor v16.16b, v16.16b, v8.16b // h2k | h1k - aese v3.16b, v31.16b // AES block 3 - round N-1 - aese v0.16b, v31.16b // AES block 0 - round N-1 - b.ge .Ldec_tail // handle tail - - ldr q4, [x0, #0] // AES block 0 - load ciphertext - ldr q5, [x0, #16] // AES block 1 - load ciphertext - rev w9, w12 // CTR block 4 - eor v0.16b, v4.16b, v0.16b // AES block 0 - result - eor v1.16b, v5.16b, v1.16b // AES block 1 - result - rev64 v5.16b, v5.16b // GHASH block 1 - ldr q7, [x0, #48] // AES block 3 - load ciphertext - mov x7, v0.d[1] // AES block 0 - mov high - mov x6, v0.d[0] // AES block 0 - mov low - rev64 v4.16b, v4.16b // GHASH block 0 - add w12, w12, #1 // CTR block 4 - fmov d0, x10 // CTR block 4 - orr x9, x11, x9, lsl #32 // CTR block 4 - fmov v0.d[1], x9 // CTR block 4 - rev w9, w12 // CTR block 5 - add w12, w12, #1 // CTR block 5 - mov x19, v1.d[0] // AES block 1 - mov low - orr x9, x11, x9, lsl #32 // CTR block 5 - mov x20, v1.d[1] // AES block 1 - mov high - eor x7, x7, x14 // AES block 0 - round N high - eor x6, x6, x13 // AES block 0 - round N low - stp x6, x7, [x2], #16 // AES block 0 - store result - fmov d1, x10 // CTR block 5 - ldr q6, [x0, #32] // AES block 2 - load ciphertext - add x0, x0, #64 // AES input_ptr update - fmov v1.d[1], x9 // CTR block 5 - rev w9, w12 // CTR block 6 - add w12, w12, #1 // CTR block 6 - eor x19, x19, x13 // AES block 1 - round N low - orr x9, x11, x9, lsl #32 // CTR block 6 - eor x20, x20, x14 // AES block 1 - round N high - stp x19, x20, [x2], #16 // AES block 1 - store result - eor v2.16b, v6.16b, v2.16b // AES block 2 - result - cmp x0, x5 // check if we have <= 8 blocks - b.ge .Ldec_prepretail // do prepretail - -.Ldec_main_loop: // main loop start - mov x21, v2.d[0] // AES block 4k+2 - mov low - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - mov x22, v2.d[1] // AES block 4k+2 - mov high - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d2, x10 // CTR block 4k+6 - fmov v2.d[1], x9 // CTR block 4k+6 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev w9, w12 // CTR block 4k+7 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - mov x24, v3.d[1] // AES block 4k+3 - mov high - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - mov x23, v3.d[0] // AES block 4k+3 - mov low - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - mov d8, v4.d[1] // GHASH block 4k - mid - fmov d3, x10 // CTR block 4k+7 - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - orr x9, x11, x9, lsl #32 // CTR block 4k+7 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - fmov v3.d[1], x9 // CTR block 4k+7 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - eor x22, x22, x14 // AES block 4k+2 - round N high - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - mov d10, v17.d[1] // GHASH block 4k - mid - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - rev64 v6.16b, v6.16b // GHASH block 4k+2 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - eor x21, x21, x13 // AES block 4k+2 - round N low - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - stp x21, x22, [x2], #16 // AES block 4k+2 - store result - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - rev64 v7.16b, v7.16b // GHASH block 4k+3 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - eor x23, x23, x13 // AES block 4k+3 - round N low - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - eor x24, x24, x14 // AES block 4k+3 - round N high - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - add w12, w12, #1 // CTR block 4k+7 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - mov d8, v6.d[1] // GHASH block 4k+2 - mid - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - rev w9, w12 // CTR block 4k+8 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - add w12, w12, #1 // CTR block 4k+8 - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - mov d6, v7.d[1] // GHASH block 4k+3 - mid - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - orr x9, x11, x9, lsl #32 // CTR block 4k+8 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - cmp x17, #12 // setup flags for AES-128/192/256 check - eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid - movi v8.8b, #0xc2 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - shl d8, d8, #56 // mod_constant - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - b.lt .Ldec_main_loop_continue // branch if AES-128 - + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq .Ldec_finish_prepretail // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 11 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - b.eq .Ldec_main_loop_continue // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - -.Ldec_main_loop_continue: - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - ldr q4, [x0, #0] // AES block 4k+4 - load ciphertext - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - ldr q5, [x0, #16] // AES block 4k+5 - load ciphertext - eor v0.16b, v4.16b, v0.16b // AES block 4k+4 - result - stp x23, x24, [x2], #16 // AES block 4k+3 - store result - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - ldr q7, [x0, #48] // AES block 4k+7 - load ciphertext - ldr q6, [x0, #32] // AES block 4k+6 - load ciphertext - mov x7, v0.d[1] // AES block 4k+4 - mov high - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - add x0, x0, #64 // AES input_ptr update - mov x6, v0.d[0] // AES block 4k+4 - mov low - fmov d0, x10 // CTR block 4k+8 - fmov v0.d[1], x9 // CTR block 4k+8 - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor v1.16b, v5.16b, v1.16b // AES block 4k+5 - result - rev w9, w12 // CTR block 4k+9 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - orr x9, x11, x9, lsl #32 // CTR block 4k+9 - cmp x0, x5 // .LOOP CONTROL - add w12, w12, #1 // CTR block 4k+9 - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high - mov x20, v1.d[1] // AES block 4k+5 - mov high - eor v2.16b, v6.16b, v2.16b // AES block 4k+6 - result - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low - mov x19, v1.d[0] // AES block 4k+5 - mov low - fmov d1, x10 // CTR block 4k+9 - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - fmov v1.d[1], x9 // CTR block 4k+9 - rev w9, w12 // CTR block 4k+10 - add w12, w12, #1 // CTR block 4k+10 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - orr x9, x11, x9, lsl #32 // CTR block 4k+10 - rev64 v5.16b, v5.16b // GHASH block 4k+5 - eor x20, x20, x14 // AES block 4k+5 - round N high - stp x6, x7, [x2], #16 // AES block 4k+4 - store result - eor x19, x19, x13 // AES block 4k+5 - round N low - stp x19, x20, [x2], #16 // AES block 4k+5 - store result - rev64 v4.16b, v4.16b // GHASH block 4k+4 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - b.lt .Ldec_main_loop - -.Ldec_prepretail: // PREPRETAIL - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - mov x21, v2.d[0] // AES block 4k+2 - mov low - eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - mov x22, v2.d[1] // AES block 4k+2 - mov high - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d2, x10 // CTR block 4k+6 - fmov v2.d[1], x9 // CTR block 4k+6 - rev w9, w12 // CTR block 4k+7 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev64 v6.16b, v6.16b // GHASH block 4k+2 - orr x9, x11, x9, lsl #32 // CTR block 4k+7 - mov x23, v3.d[0] // AES block 4k+3 - mov low - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - mov x24, v3.d[1] // AES block 4k+3 - mov high - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - mov d8, v4.d[1] // GHASH block 4k - mid - fmov d3, x10 // CTR block 4k+7 - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - fmov v3.d[1], x9 // CTR block 4k+7 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - mov d10, v17.d[1] // GHASH block 4k - mid - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - rev64 v7.16b, v7.16b // GHASH block 4k+3 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - mov d8, v6.d[1] // GHASH block 4k+2 - mid - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - mov d6, v7.d[1] // GHASH block 4k+3 - mid - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - movi v8.8b, #0xc2 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low - pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - cmp x17, #12 // setup flags for AES-128/192/256 check - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - shl d8, d8, #56 // mod_constant - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - b.lt .Ldec_finish_prepretail // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 11 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 11 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + aesmc v3.16b, v3.16b // AES block 3 - round 11 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 12 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 - b.eq .Ldec_finish_prepretail // branch if AES-192 - - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 .Ldec_finish_prepretail: - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor x22, x22, x14 // AES block 4k+2 - round N high - eor x23, x23, x13 // AES block 4k+3 - round N low - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - add w12, w12, #1 // CTR block 4k+7 - eor x21, x21, x13 // AES block 4k+2 - round N low - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor x24, x24, x14 // AES block 4k+3 - round N high - stp x21, x22, [x2], #16 // AES block 4k+2 - store result - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - stp x23, x24, [x2], #16 // AES block 4k+3 - store result - - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 .Ldec_tail: // TAIL - sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process - ld1 { v5.16b}, [x0], #16 // AES block 4k+4 - load ciphertext - eor v0.16b, v5.16b, v0.16b // AES block 4k+4 - result - mov x6, v0.d[0] // AES block 4k+4 - mov low - mov x7, v0.d[1] // AES block 4k+4 - mov high - ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ld1 { v5.16b}, [x0], #16 // AES block 0 - load ciphertext + eor v0.16b, v5.16b, v0.16b // AES block 0 - result + mov x6, v0.d[0] // AES block 0 - mov low + mov x7, v0.d[1] // AES block 0 - mov high + ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high cmp x5, #48 - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high b.gt .Ldec_blocks_more_than_3 - sub w12, w12, #1 mov v3.16b, v2.16b movi v10.8b, #0 movi v11.8b, #0 - cmp x5, #32 movi v9.8b, #0 mov v2.16b, v1.16b + cmp x5, #32 b.gt .Ldec_blocks_more_than_2 - sub w12, w12, #1 mov v3.16b, v1.16b cmp x5, #16 b.gt .Ldec_blocks_more_than_1 - sub w12, w12, #1 b .Ldec_blocks_less_than_1 .Ldec_blocks_more_than_3: // blocks left > 3 - rev64 v4.16b, v5.16b // GHASH final-3 block - ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext - stp x6, x7, [x2], #16 // AES final-3 block - store result - mov d10, v17.d[1] // GHASH final-3 block - mid - eor v4.16b, v4.16b, v8.16b // feed in partial tag - eor v0.16b, v5.16b, v1.16b // AES final-2 block - result - mov d22, v4.d[1] // GHASH final-3 block - mid - mov x6, v0.d[0] // AES final-2 block - mov low - mov x7, v0.d[1] // AES final-2 block - mov high - eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid - movi v8.8b, #0 // suppress further partial tag feed in - pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high - pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid - eor x6, x6, x13 // AES final-2 block - round N low - pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low - eor x7, x7, x14 // AES final-2 block - round N high + rev64 v4.16b, v5.16b // GHASH final-3 block + ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext + stp x6, x7, [x2], #16 // AES final-3 block - store result + mov d10, v17.d[1] // GHASH final-3 block - mid + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor v0.16b, v5.16b, v1.16b // AES final-2 block - result + mov d22, v4.d[1] // GHASH final-3 block - mid + mov x6, v0.d[0] // AES final-2 block - mov low + mov x7, v0.d[1] // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor x6, x6, x13 // AES final-2 block - round N low + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + eor x7, x7, x14 // AES final-2 block - round N high .Ldec_blocks_more_than_2: // blocks left > 2 - rev64 v4.16b, v5.16b // GHASH final-2 block - ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext - eor v4.16b, v4.16b, v8.16b // feed in partial tag - stp x6, x7, [x2], #16 // AES final-2 block - store result - eor v0.16b, v5.16b, v2.16b // AES final-1 block - result - mov d22, v4.d[1] // GHASH final-2 block - mid - pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low - pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high - eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid - mov x6, v0.d[0] // AES final-1 block - mov low - mov x7, v0.d[1] // AES final-1 block - mov high - eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low - movi v8.8b, #0 // suppress further partial tag feed in - pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high - eor x6, x6, x13 // AES final-1 block - round N low - eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid - eor x7, x7, x14 // AES final-1 block - round N high + rev64 v4.16b, v5.16b // GHASH final-2 block + ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + stp x6, x7, [x2], #16 // AES final-2 block - store result + eor v0.16b, v5.16b, v2.16b // AES final-1 block - result + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + mov x6, v0.d[0] // AES final-1 block - mov low + mov x7, v0.d[1] // AES final-1 block - mov high + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + movi v8.8b, #0 // suppress further partial tag feed in + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + eor x6, x6, x13 // AES final-1 block - round N low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + eor x7, x7, x14 // AES final-1 block - round N high .Ldec_blocks_more_than_1: // blocks left > 1 - stp x6, x7, [x2], #16 // AES final-1 block - store result - rev64 v4.16b, v5.16b // GHASH final-1 block - ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext - eor v4.16b, v4.16b, v8.16b // feed in partial tag - movi v8.8b, #0 // suppress further partial tag feed in - mov d22, v4.d[1] // GHASH final-1 block - mid - eor v0.16b, v5.16b, v3.16b // AES final block - result - pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high - eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid - pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low - mov x6, v0.d[0] // AES final block - mov low - ins v22.d[1], v22.d[0] // GHASH final-1 block - mid - mov x7, v0.d[1] // AES final block - mov high - pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid - eor x6, x6, x13 // AES final block - round N low - eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low - eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high - eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid - eor x7, x7, x14 // AES final block - round N high + stp x6, x7, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block + ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + movi v8.8b, #0 // suppress further partial tag feed in + mov d22, v4.d[1] // GHASH final-1 block - mid + eor v0.16b, v5.16b, v3.16b // AES final block - result + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + mov x6, v0.d[0] // AES final block - mov low + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + mov x7, v0.d[1] // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + eor x6, x6, x13 // AES final block - round N low + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor x7, x7, x14 // AES final block - round N high .Ldec_blocks_less_than_1: // blocks left <= 1 - and x1, x1, #127 // bit_length %= 128 - mvn x14, xzr // rkN_h = 0xffffffffffffffff - sub x1, x1, #128 // bit_length -= 128 - mvn x13, xzr // rkN_l = 0xffffffffffffffff - ldp x4, x5, [x2] // load existing bytes we need to not overwrite - neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) - and x1, x1, #127 // bit_length %= 128 - lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x14, xzr // rkN_h = 0xffffffffffffffff + sub x1, x1, #128 // bit_length -= 128 + mvn x13, xzr // rkN_l = 0xffffffffffffffff + ldp x4, x5, [x2] // load existing bytes we need to not overwrite + neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block cmp x1, #64 csel x9, x13, x14, lt csel x10, x14, xzr, lt - fmov d0, x9 // ctr0b is mask for last block + fmov d0, x9 // ctr0b is mask for last block and x6, x6, x9 mov v0.d[1], x10 - bic x4, x4, x9 // mask out low existing bytes - rev w9, w12 - bic x5, x5, x10 // mask out high existing bytes + bic x4, x4, x9 // mask out low existing bytes + bic x5, x5, x10 // mask out high existing bytes orr x6, x6, x4 and x7, x7, x10 orr x7, x7, x5 - and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits - rev64 v4.16b, v5.16b // GHASH final block - eor v4.16b, v4.16b, v8.16b // feed in partial tag - pmull v21.1q, v4.1d, v12.1d // GHASH final block - low - mov d8, v4.d[1] // GHASH final block - mid - eor v8.8b, v8.8b, v4.8b // GHASH final block - mid - pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high - pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final block - high - eor v11.16b, v11.16b, v21.16b // GHASH final block - low - eor v10.16b, v10.16b, v8.16b // GHASH final block - mid - movi v8.8b, #0xc2 - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - shl d8, d8, #56 // mod_constant - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low + and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits + rev64 v4.16b, v5.16b // GHASH final block + eor v4.16b, v4.16b, v8.16b // feed in partial tag + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + mov d8, v4.d[1] // GHASH final block - mid + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + ldr d8, [sp, #128] + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b // MODULO - fold into low stp x6, x7, [x2] - str w9, [x16, #12] // store the updated counter - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + rev64 v11.16b, v11.16b // Final Tag mov x0, x15 - st1 { v11.16b }, [x3] + st1 { v11.16b }, [x3] // Store final tag ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x23, x24, [sp, #48] @@ -1546,9 +1468,1440 @@ ldp d10, d11, [sp, #80] ldp d12, d13, [sp, #96] ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 + ldp x29, x30, [sp], #224 AARCH64_VALIDATE_LINK_REGISTER ret .size aes_gcm_dec_kernel,.-aes_gcm_dec_kernel -#endif + +.globl aes_gcm_enc_kernel_eor3 +.hidden aes_gcm_enc_kernel_eor3 +.type aes_gcm_enc_kernel_eor3,%function +.align 4 +aes_gcm_enc_kernel_eor3: + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + ld1 { v0.16b}, [x4] // .Load initial counter block + stp x19, x20, [sp, #16] + mov v1.16b, v0.16b // Initialize ctr1-3 from ctr0 + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 // Pointer to counter block in memory + mov x8, x5 // Pointer to AES key schedule context + stp x21, x22, [sp, #32] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel_eor3 + stp d8, d9, [sp, #64] // Save Neon registers + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr w17, [x8, #240] // .Load number of AES rounds + add x7, x8, x17, lsl #4 // Calculate pointer to the last round key + ldp x13, x14, [x7] // load round N key (for final XOR) + ldr q31, [x7, #-16] // load round N-1 key + add x4, x0, x1, lsr #3 // Calculate end of input + lsr x5, x1, #3 // Total byte length + mov x15, x5 + ldr w12, [x16, #12] // .Load counter's low 32 bits + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes + add x5, x5, x0 + rev w12, w12 // Reverse for big-endian increment + uxtw x10, w12 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov x21, #0xc200000000000000 // GHASH reduction constant + str x21, [sp, #128] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., 160 + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str q0, [sp, #160] // Store base counter for block 0-3 + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev w20, w12 + mov v0.s[3], w20 // ctr0 + 0 + add w20, w12, #1 + rev w20, w20 + mov v1.s[3], w20 // ctr0 + 1 + add w20, w12, #2 + rev w20, w20 + mov v2.s[3], w20 // ctr0 + 2 + add w20, w12, #3 + rev w20, w20 + mov v3.s[3], w20 // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add w20, w12, #4 + rev w20, w20 + str w20, [sp, #172] // ctr0 + 4 for next iter + add w20, w12, #5 + rev w20, w20 + str w20, [sp, #188] // ctr0 + 5 for next iter + add w20, w12, #6 + rev w20, w20 + str w20, [sp, #204] // ctr0 + 6 for next iter + add w20, w12, #7 + rev w20, w20 + str w20, [sp, #220] // ctr0 + 7 for next iter + add w12, w12, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load H2, H3 (GHASH keys) + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 // Byte swap H3 for GHASH + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 // Byte swap H2 for GHASH + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + ldr q15, [x6, #80] // load H4 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 // Byte swap H4 for GHASH + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] // .Load initial GHASH accumulator (T) + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap T for GHASH + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b // Correct byte order within 64-bit lanes + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // Karatsuba key: H4_low | H3_low + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load H1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 // Byte swap H1 for GHASH + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // Karatsuba key: H4_high | H3_high + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // Karatsuba key: H2_low | H1_low + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + ldr q30, [x7] // Preload round N key for final EOR + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt .Lenc_finish_first_blocks_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + b.eq .Lenc_finish_first_blocks_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 +.Lenc_finish_first_blocks_eor3: + cmp x0, x5 // check if we have <= 4 blocks to process in the tail + eor v17.16b, v17.16b, v9.16b // Karatsuba key: H3^H4 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // Karatsuba key: H2_high | H1_high + eor v16.16b, v16.16b, v8.16b // Karatsuba key: H1^H2 + b.ge .Lenc_tail_eor3 // handle tail if no more full 4-block sets + ldp q6, q7, [x0, #32] // AES blocks 2,3 load plaintext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + eor3 v4.16b, v4.16b, v30.16b, v0.16b // AES block 0 - result = PT ^ AES(ctr0) + eor3 v5.16b, v5.16b, v30.16b, v1.16b // AES block 1 - result = PT ^ AES(ctr1) + eor3 v6.16b, v6.16b, v30.16b, v2.16b // AES block 2 - result = PT ^ AES(ctr2) + eor3 v7.16b, v7.16b, v30.16b, v3.16b // AES block 3 - result = PT ^ AES(ctr3) + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // Prepare and store counter values for the third iteration + rev w20, w12 + str w20, [sp, #172] // ctr + 8 + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] // ctr + 9 + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] // ctr + 10 + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] // ctr + 11 + add w12, w12, #4 // Advance counter base + cmp x0, x5 // check if we have <= 4 blocks remaining + b.ge .Lenc_prepretail_eor3 // go to prepretail if < 2 full loops left +.Lenc_main_loop_eor3: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + ldr d8, [sp, #128] // .Load GHASH reduction constant + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 v4.16b, v4.16b // GHASH block 4k - Byte swap CT + rev64 v5.16b, v5.16b // GHASH block 4k+1 - Byte swap CT + rev64 v6.16b, v6.16b // GHASH block 4k+2 - Byte swap CT + rev64 v7.16b, v7.16b // GHASH block 4k+3 - Byte swap CT + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // GHASH - prepare acc for XOR + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid Karatsuba key + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d20, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v20.8b, v20.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + mov d21, v5.d[1] // GHASH block 4k+1 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + eor v21.8b, v21.8b, v5.8b // GHASH block 4k+1 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + pmull v10.1q, v20.1d, v10.1d // GHASH block 4k - mid + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + pmull v21.1q, v21.1d, v17.1d // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + eor v10.16b, v10.16b, v21.16b // GHASH block 4k+1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + ext v22.16b, v22.16b, v6.16b, #8 // GHASH block 4k+2 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v22.16b, v22.16b, v6.16b // GHASH block 4k+2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull2 v22.1q, v22.2d, v16.2d // GHASH block 4k+2 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + mov d23, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v23.8b, v23.8b, v7.8b // GHASH block 4k+3 - mid + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + pmull v23.1q, v23.1d, v16.1d // GHASH block 4k+3 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor3 v10.16b, v10.16b, v22.16b, v23.16b // GHASH block 4k+2/3 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v22.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v21.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v22.16b, v22.16b, v21.16b // GHASH block 4k+3 - high + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull2 v23.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + pmull v21.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull v20.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + eor3 v9.16b, v9.16b, v22.16b, v23.16b // GHASH block 4k/1/2/3 - high + pmull v22.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + ldp q6, q7, [x0, #32] + ldp q4, q5, [x0], #64 + eor v20.16b, v20.16b, v21.16b // GHASH block 4k+1 - low + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + eor3 v11.16b, v11.16b, v22.16b, v20.16b // GHASH block 4k/1/2/3 - low + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + eor3 v10.16b, v10.16b, v9.16b, v11.16b // MODULO - karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v20.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt .Lenc_main_loop_continue_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq .Lenc_main_loop_continue_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +.Lenc_main_loop_continue_eor3: + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v20.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v20.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v9.16b, v11.16b, v20.16b // MODULO - fold into low + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor3 v4.16b, v4.16b, v30.16b, v0.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor3 v5.16b, v5.16b, v30.16b, v1.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor3 v6.16b, v6.16b, v30.16b, v2.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor3 v7.16b, v7.16b, v30.16b, v3.16b // AES block 4k+7 - result + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // We used these registers as temporaries above so reload the RKs. + ldp q20, q21, [x8, #32] // load rk2, rk3 + ldp q22, q23, [x8, #64] // load rk4, rk5 + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + rev w20, w12 + str w20, [sp, #172] + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] + add w12, w12, #4 + cmp x0, x5 // .LOOP_eor3 CONTROL + b.lt .Lenc_main_loop_eor3 +.Lenc_prepretail_eor3: // PREPRETAIL + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + rev64 v6.16b, v6.16b // GHASH block 2 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + rev64 v4.16b, v4.16b // GHASH block 0 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v5.16b, v5.16b // GHASH block 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid Karatsuba key + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + mov d4, v5.d[1] // GHASH block 1 - mid + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + mov d8, v6.d[1] // GHASH block 2 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + ins v8.d[1], v8.d[0] // GHASH block 2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v9.16b, v9.16b, v4.16b // GHASH block 2 - high + mov d4, v7.d[1] // GHASH block 3 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + eor v4.8b, v4.8b, v7.8b // GHASH block 3 - mid + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull v4.1q, v4.1d, v16.1d // GHASH block 3 - mid + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + eor v9.16b, v9.16b, v5.16b // GHASH block 3 - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + ldr d8, [sp, #128] + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + eor v10.16b, v10.16b, v4.16b // GHASH block 3 - mid + pmull v6.1q, v7.1d, v12.1d // GHASH block 3 - low + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + eor v11.16b, v11.16b, v6.16b // GHASH block 3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v10.16b, v10.16b, v9.16b // karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + pmull v4.1q, v9.1d, v8.1d + ext v9.16b, v9.16b, v9.16b, #8 + eor v10.16b, v10.16b, v11.16b + b.lt .Lenc_finish_prepretail_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq .Lenc_finish_prepretail_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +.Lenc_finish_prepretail_eor3: + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + eor3 v10.16b, v10.16b, v4.16b, v9.16b + pmull v4.1q, v10.1d, v8.1d + ext v10.16b, v10.16b, v10.16b, #8 + eor3 v11.16b, v11.16b, v4.16b, v10.16b +.Lenc_tail_eor3: // TAIL: Process remaining 0 to 3 blocks + ext v8.16b, v11.16b, v11.16b, #8 // Save current GHASH state for partial tag feed-in + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ldp x6, x7, [x0], #16 // AES block 0 - load plaintext + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high + cmp x5, #48 + fmov d4, x6 // AES block 0 - mov low + fmov v4.d[1], x7 // AES block 0 - mov high + eor v5.16b, v4.16b, v0.16b // AES block 0 - result + b.gt .Lenc_blocks_more_than_3_eor3 + cmp x5, #32 + mov v3.16b, v2.16b + movi v11.8b, #0 + movi v9.8b, #0 + mov v2.16b, v1.16b + movi v10.8b, #0 + b.gt .Lenc_blocks_more_than_2_eor3 + mov v3.16b, v1.16b + cmp x5, #16 + b.gt .Lenc_blocks_more_than_1_eor3 + b .Lenc_blocks_less_than_1_eor3 +.Lenc_blocks_more_than_3_eor3: // blocks left > 3 + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-3 block + eor x6, x6, x13 // AES final-2 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor x7, x7, x14 // AES final-2 block - round N high + mov d22, v4.d[1] // GHASH final-3 block - mid + fmov d5, x6 // AES final-2 block - mov low + fmov v5.d[1], x7 // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + mov d10, v17.d[1] // GHASH final-3 block - mid + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor v5.16b, v5.16b, v1.16b // AES final-2 block - result +.Lenc_blocks_more_than_2_eor3: // blocks left > 2 + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-2 block + eor x6, x6, x13 // AES final-1 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + fmov d5, x6 // AES final-1 block - mov low + eor x7, x7, x14 // AES final-1 block - round N high + fmov v5.d[1], x7 // AES final-1 block - mov high + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + eor v5.16b, v5.16b, v2.16b // AES final-1 block - result + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid +.Lenc_blocks_more_than_1_eor3: // blocks left > 1 + st1 { v5.16b}, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block: Byte Swap CT + ldp x6, x7, [x0], #16 // AES final block - load plaintext + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + movi v8.8b, #0 // Clear for next block + eor x6, x6, x13 // AES final block - round N low + mov d22, v4.d[1] // GHASH final-1 block - mid + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor x7, x7, x14 // AES final block - round N high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + fmov d5, x6 // AES final block - mov low + fmov v5.d[1], x7 // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + eor v5.16b, v5.16b, v3.16b // AES final block - result + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low +.Lenc_blocks_less_than_1_eor3: // .Last partial block handling + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x13, xzr // Mask for low 64 bits + sub x1, x1, #128 // + neg x1, x1 // Valid bits in the last block (1-128) + ldr q18, [x2] // .Load destination for merging + mvn x14, xzr // Mask for high 64 bits + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + cmp x1, #64 + csel x6, x13, x14, lt + csel x7, x14, xzr, lt + fmov d0, x6 // ctr0d is mask for last block + fmov v0.d[1], x7 + and v5.16b, v5.16b, v0.16b // Mask out unused bits of the last CT block + rev64 v4.16b, v5.16b // GHASH final block - byte swap + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + bif v5.16b, v18.16b, v0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + mov d8, v4.d[1] // GHASH final block - mid + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + fmov d8, x21 + eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v7.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + st1 { v5.16b}, [x2] // store all 16B + eor3 v11.16b, v11.16b, v9.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap GHASH result + rev64 v11.16b, v11.16b // Final Tag + mov x0, x15 + st1 { v11.16b }, [x3] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size aes_gcm_enc_kernel_eor3,.-aes_gcm_enc_kernel_eor3 +.globl aes_gcm_dec_kernel_eor3 +.hidden aes_gcm_dec_kernel_eor3 +.type aes_gcm_dec_kernel_eor3,%function +.align 4 +aes_gcm_dec_kernel_eor3: + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + stp x19, x20, [sp, #16] + ld1 { v0.16b}, [x4] + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 + mov x8, x5 + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp d8, d9, [sp, #64] + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr w17, [x8, #240] // .Load number of AES rounds + add x19, x8, x17, lsl #4 // borrow input_l1 for last key + ldp x13, x14, [x19] // load round N keys + ldr q31, [x19, #-16] // load round N-1 keys + add x4, x0, x1, lsr #3 // end_input_ptr + lsr x5, x1, #3 // byte_len + mov x15, x5 + ldr w9, [x16, #12] // .Load scalar 32-bit counter (CTR) + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + add x5, x5, x0 + rev w9, w9 // Reverse it once for big-endian incrementing + uxtw x10, w9 // Zero extend reversed w9 into x10 + str q0, [sp, #160] + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + rev w20, w9 + mov v0.s[3], w20 + add w20, w9, #1 + rev w20, w20 + mov v1.s[3], w20 + add w20, w9, #2 + rev w20, w20 + mov v2.s[3], w20 + add w20, w9, #3 + rev w20, w20 + mov v3.s[3], w20 + add w20, w9, #4 + rev w20, w20 + str w20, [sp, #172] + add w20, w9, #5 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #6 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #7 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov x21, #0xc200000000000000 + str x21, [sp, #128] + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load h2, h3 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 + ldr q15, [x6, #80] // load h4 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // h4l | h3l + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load h1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // h2l | h1l + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_finish_first_blocks_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq .Ldec_finish_first_blocks_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +.Ldec_finish_first_blocks_eor3: + ldr q27, [x19] // load rkN + cmp x0, x5 // check if we have <= 4 blocks + eor v17.16b, v17.16b, v9.16b // h4k | h3k + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // h2h | h1h + eor v16.16b, v16.16b, v8.16b // h2k | h1k + b.ge .Ldec_tail_eor3 // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value (v30). This avoids several fmovs. + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + eor3 v0.16b, v4.16b, v0.16b, v27.16b // AES block 0 - result + eor3 v1.16b, v5.16b, v1.16b, v27.16b // AES block 1 - result + eor3 v2.16b, v6.16b, v2.16b, v27.16b // AES block 2 - result + eor3 v3.16b, v7.16b, v3.16b, v27.16b // AES block 3 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 0-3 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 // check if we have <= 4 blocks + b.ge .Ldec_prepretail_eor3 // do prepretail +.Ldec_main_loop_eor3: // main loop start + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + rev64 v4.16b, v4.16b // GHASH block 4k + rev64 v5.16b, v5.16b // GHASH block 4k+1 + rev64 v6.16b, v6.16b // GHASH block 4k+2 + rev64 v7.16b, v7.16b // GHASH block 4k+3 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // PRE 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d8, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + mov d4, v5.d[1] // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor3 v11.16b, v11.16b, v8.16b, v5.16b // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + mov d8, v6.d[1] // GHASH block 4k+2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid + ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v28.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + mov d6, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull v27.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + eor3 v10.16b, v10.16b, v4.16b, v8.16b // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid + eor3 v9.16b, v9.16b, v28.16b, v5.16b // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid + ldr d8, [sp, #128] + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + eor3 v10.16b, v10.16b, v6.16b, v7.16b // GHASH block 4k+3 - mid & MODULO - fold into mid + eor v11.16b, v11.16b, v27.16b // GHASH block 4k+3 - low + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v6.16b, v9.16b // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v11.16b, v8.16b, v10.16b // MODULO - fold into low + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_main_loop_continue_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq .Ldec_main_loop_continue_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +.Ldec_main_loop_continue_eor3: + ldr q27, [x19] // load rkN + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor3 v0.16b, v4.16b, v0.16b, v27.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor3 v1.16b, v5.16b, v1.16b, v27.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor3 v2.16b, v6.16b, v2.16b, v27.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor3 v3.16b, v7.16b, v3.16b, v27.16b // AES block 4k+7 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 + b.lt .Ldec_main_loop_eor3 +.Ldec_prepretail_eor3: // PREPRETAIL + rev64 v4.16b, v4.16b // GHASH block 0 + rev64 v5.16b, v5.16b // GHASH block 1 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v6.16b, v6.16b // GHASH block 2 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + mov d4, v5.d[1] // GHASH block 1 - mid + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + mov d8, v6.d[1] // GHASH block 2 - mid + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + ins v8.d[1], v8.d[0] // GHASH block 2 - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + eor3 v9.16b, v9.16b, v4.16b, v5.16b // GHASH block 2 - high & GHASH block 3 - high + pmull v4.1q, v7.1d, v12.1d // GHASH block 3 - low + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + mov d6, v7.d[1] // GHASH block 3 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v6.8b, v6.8b, v7.8b // GHASH block 3 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + movi v8.8b, #0xc2 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v11.16b, v11.16b, v4.16b // GHASH block 3 - low + pmull v6.1q, v6.1d, v16.1d // GHASH block 3 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + eor v10.16b, v10.16b, v6.16b // GHASH block 3 - mid + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + shl d8, d8, #56 // mod_constant + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt .Ldec_finish_prepretail_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq .Ldec_finish_prepretail_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +.Ldec_finish_prepretail_eor3: + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v7.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v11.16b, v8.16b, v10.16b // MODULO - fold into low + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 +.Ldec_tail_eor3: // TAIL + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ld1 { v5.16b}, [x0], #16 // AES block 0 - load ciphertext + eor v0.16b, v5.16b, v0.16b // AES block 0 - result + mov x6, v0.d[0] // AES block 0 - mov low + mov x7, v0.d[1] // AES block 0 - mov high + ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high + cmp x5, #48 + b.gt .Ldec_blocks_more_than_3_eor3 + mov v3.16b, v2.16b + movi v10.8b, #0 + movi v11.8b, #0 + movi v9.8b, #0 + mov v2.16b, v1.16b + cmp x5, #32 + b.gt .Ldec_blocks_more_than_2_eor3 + mov v3.16b, v1.16b + cmp x5, #16 + b.gt .Ldec_blocks_more_than_1_eor3 + b .Ldec_blocks_less_than_1_eor3 +.Ldec_blocks_more_than_3_eor3: // blocks left > 3 + rev64 v4.16b, v5.16b // GHASH final-3 block + ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext + stp x6, x7, [x2], #16 // AES final-3 block - store result + mov d10, v17.d[1] // GHASH final-3 block - mid + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor v0.16b, v5.16b, v1.16b // AES final-2 block - result + mov d22, v4.d[1] // GHASH final-3 block - mid + mov x6, v0.d[0] // AES final-2 block - mov low + mov x7, v0.d[1] // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor x6, x6, x13 // AES final-2 block - round N low + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + eor x7, x7, x14 // AES final-2 block - round N high +.Ldec_blocks_more_than_2_eor3: // blocks left > 2 + rev64 v4.16b, v5.16b // GHASH final-2 block + ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + stp x6, x7, [x2], #16 // AES final-2 block - store result + eor v0.16b, v5.16b, v2.16b // AES final-1 block - result + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + mov x6, v0.d[0] // AES final-1 block - mov low + mov x7, v0.d[1] // AES final-1 block - mov high + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + movi v8.8b, #0 // suppress further partial tag feed in + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + eor x6, x6, x13 // AES final-1 block - round N low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + eor x7, x7, x14 // AES final-1 block - round N high +.Ldec_blocks_more_than_1_eor3: // blocks left > 1 + stp x6, x7, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block + ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + movi v8.8b, #0 // suppress further partial tag feed in + mov d22, v4.d[1] // GHASH final-1 block - mid + eor v0.16b, v5.16b, v3.16b // AES final block - result + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + mov x6, v0.d[0] // AES final block - mov low + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + mov x7, v0.d[1] // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + eor x6, x6, x13 // AES final block - round N low + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor x7, x7, x14 // AES final block - round N high +.Ldec_blocks_less_than_1_eor3: // blocks left <= 1 + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x14, xzr // rkN_h = 0xffffffffffffffff + sub x1, x1, #128 // bit_length -= 128 + mvn x13, xzr // rkN_l = 0xffffffffffffffff + ldp x4, x5, [x2] // load existing bytes we need to not overwrite + neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + cmp x1, #64 + csel x9, x13, x14, lt + csel x10, x14, xzr, lt + fmov d0, x9 // ctr0b is mask for last block + and x6, x6, x9 + mov v0.d[1], x10 + bic x4, x4, x9 // mask out low existing bytes + bic x5, x5, x10 // mask out high existing bytes + orr x6, x6, x4 + and x7, x7, x10 + orr x7, x7, x5 + and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits + rev64 v4.16b, v5.16b // GHASH final block + eor v4.16b, v4.16b, v8.16b // feed in partial tag + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + mov d8, v4.d[1] // GHASH final block - mid + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + ldr d8, [sp, #128] + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b // MODULO - fold into low + stp x6, x7, [x2] + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 + rev64 v11.16b, v11.16b // Final Tag + mov x0, x15 + st1 { v11.16b }, [x3] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size aes_gcm_dec_kernel_eor3,.-aes_gcm_dec_kernel_eor3 +#endif // __ARM_MAX_ARCH__ >= 8 #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__ELF__)
diff --git a/gen/bcm/aesv8-gcm-armv8-win.S b/gen/bcm/aesv8-gcm-armv8-win.S index ce27fd3..a5a2b42 100644 --- a/gen/bcm/aesv8-gcm-armv8-win.S +++ b/gen/bcm/aesv8-gcm-armv8-win.S
@@ -5,8 +5,7 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(_WIN32) #if __ARM_MAX_ARCH__ >= 8 - -.arch armv8-a+crypto +.arch armv8.2-a+crypto+sha3 .text .globl aes_gcm_enc_kernel @@ -16,765 +15,731 @@ .align 4 aes_gcm_enc_kernel: AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! + stp x29, x30, [sp, #-224]! mov x29, sp + ld1 { v0.16b}, [x4] // Load initial counter block stp x19, x20, [sp, #16] - mov x16, x4 - mov x8, x5 + mov v1.16b, v0.16b // Initialize ctr1-3 from ctr0 + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 // Pointer to counter block in memory + mov x8, x5 // Pointer to AES key schedule context stp x21, x22, [sp, #32] - stp x23, x24, [sp, #48] - stp d8, d9, [sp, #64] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel + stp d8, d9, [sp, #64] // Save Neon registers stp d10, d11, [sp, #80] stp d12, d13, [sp, #96] stp d14, d15, [sp, #112] - ldr w17, [x8, #240] - add x19, x8, x17, lsl #4 // borrow input_l1 for last key - ldp x13, x14, [x19] // load round N keys - ldr q31, [x19, #-16] // load round N-1 keys - add x4, x0, x1, lsr #3 // end_input_ptr - lsr x5, x1, #3 // byte_len + ldr w17, [x8, #240] // Load number of AES rounds + add x7, x8, x17, lsl #4 // Calculate pointer to the last round key + ldp x13, x14, [x7] // load round N key (for final XOR) + ldr q31, [x7, #-16] // load round N-1 key + add x4, x0, x1, lsr #3 // Calculate end of input + lsr x5, x1, #3 // Total byte length mov x15, x5 - ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 - ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible - sub x5, x5, #1 // byte_len - 1 - ldr q18, [x8, #0] // load rk0 - and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - ldr q25, [x8, #112] // load rk7 + ldr w12, [x16, #12] // Load counter's low 32 bits + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes add x5, x5, x0 - lsr x12, x11, #32 - fmov d2, x10 // CTR block 2 - orr w11, w11, w11 - rev w12, w12 // rev_ctr32 - fmov d1, x10 // CTR block 1 + rev w12, w12 // Reverse for big-endian increment + uxtw x10, w12 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov x21, #0xc200000000000000 // GHASH reduction constant + str x21, [sp, #128] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., 160 + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str q0, [sp, #160] // Store base counter for block 0-3 + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev w20, w12 + mov v0.s[3], w20 // ctr0 + 0 + add w20, w12, #1 + rev w20, w20 + mov v1.s[3], w20 // ctr0 + 1 + add w20, w12, #2 + rev w20, w20 + mov v2.s[3], w20 // ctr0 + 2 + add w20, w12, #3 + rev w20, w20 + mov v3.s[3], w20 // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add w20, w12, #4 + rev w20, w20 + str w20, [sp, #172] // ctr0 + 4 for next iter + add w20, w12, #5 + rev w20, w20 + str w20, [sp, #188] // ctr0 + 5 for next iter + add w20, w12, #6 + rev w20, w20 + str w20, [sp, #204] // ctr0 + 6 for next iter + add w20, w12, #7 + rev w20, w20 + str w20, [sp, #220] // ctr0 + 7 for next iter + add w12, w12, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 0 - round 0 - add w12, w12, #1 // increment rev_ctr32 - rev w9, w12 // CTR block 1 - fmov d3, x10 // CTR block 3 - orr x9, x11, x9, lsl #32 // CTR block 1 - add w12, w12, #1 // CTR block 1 - ldr q19, [x8, #16] // load rk1 - fmov v1.d[1], x9 // CTR block 1 - rev w9, w12 // CTR block 2 - add w12, w12, #1 // CTR block 2 - orr x9, x11, x9, lsl #32 // CTR block 2 - ldr q20, [x8, #32] // load rk2 - fmov v2.d[1], x9 // CTR block 2 - rev w9, w12 // CTR block 3 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 0 - round 1 - orr x9, x11, x9, lsl #32 // CTR block 3 - fmov v3.d[1], x9 // CTR block 3 + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 1 - round 0 - ldr q21, [x8, #48] // load rk3 - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 0 - round 2 - ldr q24, [x8, #96] // load rk6 + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 2 - round 0 - ldr q23, [x8, #80] // load rk5 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 1 - round 1 - ldr q14, [x6, #48] // load h3l | h3h - ext v14.16b, v14.16b, v14.16b, #8 + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 3 - round 0 + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load H2, H3 (GHASH keys) + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 2 - round 1 - ldr q22, [x8, #64] // load rk4 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 1 - round 2 - ldr q13, [x6, #32] // load h2l | h2h - ext v13.16b, v13.16b, v13.16b, #8 + aesmc v2.16b, v2.16b // AES block 2 - round 1 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 3 - round 1 - ldr q30, [x8, #192] // load rk12 + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 // Byte swap H3 for GHASH + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 // Byte swap H2 for GHASH aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 2 - round 2 - ldr q15, [x6, #80] // load h4l | h4h - ext v15.16b, v15.16b, v15.16b, #8 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 1 - round 3 - ldr q29, [x8, #176] // load rk11 + aesmc v2.16b, v2.16b // AES block 2 - round 2 + ldr q15, [x6, #80] // load H4 aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 3 - round 2 - ldr q26, [x8, #128] // load rk8 - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 2 - round 3 - add w12, w12, #1 // CTR block 3 + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 // Byte swap H4 for GHASH aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 0 - round 3 + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] // Load initial GHASH accumulator (T) + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap T for GHASH + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b // Correct byte order within 64-bit lanes aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 3 - round 3 - ld1 { v11.16b}, [x3] - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 2 - round 4 + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // Karatsuba key: H4_low | H3_low aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 0 - round 4 + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load H1 aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 1 - round 4 + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 // Byte swap H1 for GHASH + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // Karatsuba key: H4_high | H3_high aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 3 - round 4 - cmp x17, #12 // setup flags for AES-128/192/256 check + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // Karatsuba key: H2_low | H1_low aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 0 - round 5 + aesmc v0.16b, v0.16b // AES block 0 - round 5 + ldr q30, [x7] // Preload round N key for final EOR aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 1 - round 5 + aesmc v1.16b, v1.16b // AES block 1 - round 5 aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 3 - round 5 + aesmc v3.16b, v3.16b // AES block 3 - round 5 aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 2 - round 5 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 1 - round 6 - trn2 v17.2d, v14.2d, v15.2d // h4l | h3l - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 3 - round 6 - ldr q27, [x8, #144] // load rk9 + aesmc v2.16b, v2.16b // AES block 2 - round 5 aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 0 - round 6 - ldr q12, [x6] // load h1l | h1h - ext v12.16b, v12.16b, v12.16b, #8 + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 2 - round 6 - ldr q28, [x8, #160] // load rk10 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 1 - round 7 - trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 0 - round 7 + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 2 - round 7 + aesmc v2.16b, v2.16b // AES block 2 - round 7 aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 3 - round 7 - trn2 v16.2d, v12.2d, v13.2d // h2l | h1l - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 1 - round 8 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 2 - round 8 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 3 - round 8 + aesmc v3.16b, v3.16b // AES block 3 - round 7 aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 0 - round 8 - b.lt Lenc_finish_first_blocks // branch if AES-128 - + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_finish_first_blocks // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 1 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 2 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 9 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 3 - round 9 + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 0 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 1 - round 10 + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 2 - round 10 + aesmc v2.16b, v2.16b // AES block 2 - round 10 aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 3 - round 10 + aesmc v3.16b, v3.16b // AES block 3 - round 10 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 0 - round 10 - b.eq Lenc_finish_first_blocks // branch if AES-192 - - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 1 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 2 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 0 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 3 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 1 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 2 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 0 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 3 - round 12 - + aesmc v0.16b, v0.16b // AES block 0 - round 10 + b.eq Lenc_finish_first_blocks // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 Lenc_finish_first_blocks: - cmp x0, x5 // check if we have <= 4 blocks - eor v17.16b, v17.16b, v9.16b // h4k | h3k - aese v2.16b, v31.16b // AES block 2 - round N-1 - trn1 v8.2d, v12.2d, v13.2d // h2h | h1h - aese v1.16b, v31.16b // AES block 1 - round N-1 - aese v0.16b, v31.16b // AES block 0 - round N-1 - aese v3.16b, v31.16b // AES block 3 - round N-1 - eor v16.16b, v16.16b, v8.16b // h2k | h1k - b.ge Lenc_tail // handle tail - - ldp x19, x20, [x0, #16] // AES block 1 - load plaintext - rev w9, w12 // CTR block 4 - ldp x6, x7, [x0, #0] // AES block 0 - load plaintext - ldp x23, x24, [x0, #48] // AES block 3 - load plaintext - ldp x21, x22, [x0, #32] // AES block 2 - load plaintext - add x0, x0, #64 // AES input_ptr update - eor x19, x19, x13 // AES block 1 - round N low - eor x20, x20, x14 // AES block 1 - round N high - fmov d5, x19 // AES block 1 - mov low - eor x6, x6, x13 // AES block 0 - round N low - eor x7, x7, x14 // AES block 0 - round N high - eor x24, x24, x14 // AES block 3 - round N high - fmov d4, x6 // AES block 0 - mov low - cmp x0, x5 // check if we have <= 8 blocks - fmov v4.d[1], x7 // AES block 0 - mov high - eor x23, x23, x13 // AES block 3 - round N low - eor x21, x21, x13 // AES block 2 - round N low - fmov v5.d[1], x20 // AES block 1 - mov high - fmov d6, x21 // AES block 2 - mov low - add w12, w12, #1 // CTR block 4 - orr x9, x11, x9, lsl #32 // CTR block 4 - fmov d7, x23 // AES block 3 - mov low - eor x22, x22, x14 // AES block 2 - round N high - fmov v6.d[1], x22 // AES block 2 - mov high - eor v4.16b, v4.16b, v0.16b // AES block 0 - result - fmov d0, x10 // CTR block 4 - fmov v0.d[1], x9 // CTR block 4 - rev w9, w12 // CTR block 5 - add w12, w12, #1 // CTR block 5 - eor v5.16b, v5.16b, v1.16b // AES block 1 - result - fmov d1, x10 // CTR block 5 - orr x9, x11, x9, lsl #32 // CTR block 5 - fmov v1.d[1], x9 // CTR block 5 - rev w9, w12 // CTR block 6 - st1 { v4.16b}, [x2], #16 // AES block 0 - store result - fmov v7.d[1], x24 // AES block 3 - mov high - orr x9, x11, x9, lsl #32 // CTR block 6 - eor v6.16b, v6.16b, v2.16b // AES block 2 - result - st1 { v5.16b}, [x2], #16 // AES block 1 - store result - add w12, w12, #1 // CTR block 6 - fmov d2, x10 // CTR block 6 - fmov v2.d[1], x9 // CTR block 6 - st1 { v6.16b}, [x2], #16 // AES block 2 - store result - rev w9, w12 // CTR block 7 - orr x9, x11, x9, lsl #32 // CTR block 7 - eor v7.16b, v7.16b, v3.16b // AES block 3 - result - st1 { v7.16b}, [x2], #16 // AES block 3 - store result - b.ge Lenc_prepretail // do prepretail - -Lenc_main_loop: // main loop start + cmp x0, x5 // check if we have <= 4 blocks to process in the tail + eor v17.16b, v17.16b, v9.16b // Karatsuba key: H3^H4 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // Karatsuba key: H2_high | H1_high + eor v16.16b, v16.16b, v8.16b // Karatsuba key: H1^H2 + b.ge Lenc_tail // handle tail if no more full 4-block sets + ldp q6, q7, [x0, #32] // AES blocks 2,3 load plaintext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + eor v4.16b, v4.16b, v30.16b + eor v4.16b, v4.16b, v0.16b // AES block 0 - result = PT ^ AES(ctr0) + eor v5.16b, v5.16b, v30.16b + eor v5.16b, v5.16b, v1.16b // AES block 1 - result = PT ^ AES(ctr1) + eor v6.16b, v6.16b, v30.16b + eor v6.16b, v6.16b, v2.16b // AES block 2 - result = PT ^ AES(ctr2) + eor v7.16b, v7.16b, v30.16b + eor v7.16b, v7.16b, v3.16b // AES block 3 - result = PT ^ AES(ctr3) + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // Prepare and store counter values for the third iteration + rev w20, w12 + str w20, [sp, #172] // ctr + 8 + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] // ctr + 9 + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] // ctr + 10 + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] // ctr + 11 + add w12, w12, #4 // Advance counter base + cmp x0, x5 // check if we have <= 4 blocks remaining + b.ge Lenc_prepretail // go to prepretail if < 2 full loops left +Lenc_main_loop: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d3, x10 // CTR block 4k+3 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - fmov v3.d[1], x9 // CTR block 4k+3 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - ldp x23, x24, [x0, #48] // AES block 4k+7 - load plaintext - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - ldp x21, x22, [x0, #32] // AES block 4k+6 - load plaintext - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - eor v4.16b, v4.16b, v11.16b // PRE 1 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - eor x23, x23, x13 // AES block 4k+7 - round N low - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - mov d10, v17.d[1] // GHASH block 4k - mid - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - eor x22, x22, x14 // AES block 4k+6 - round N high - mov d8, v4.d[1] // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + ldr d8, [sp, #128] // Load GHASH reduction constant + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 v4.16b, v4.16b // GHASH block 4k - Byte swap CT + rev64 v5.16b, v5.16b // GHASH block 4k+1 - Byte swap CT + rev64 v6.16b, v6.16b // GHASH block 4k+2 - Byte swap CT + rev64 v7.16b, v7.16b // GHASH block 4k+3 - Byte swap CT + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // GHASH - prepare acc for XOR + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid Karatsuba key + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d20, v4.d[1] // GHASH block 4k - mid aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - mov d8, v6.d[1] // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v20.8b, v20.8b, v4.8b // GHASH block 4k - mid aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + mov d21, v5.d[1] // GHASH block 4k+1 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + eor v21.8b, v21.8b, v5.8b // GHASH block 4k+1 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + pmull v10.1q, v20.1d, v10.1d // GHASH block 4k - mid aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + pmull v21.1q, v21.1d, v17.1d // GHASH block 4k+1 - mid aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + eor v10.16b, v10.16b, v21.16b // GHASH block 4k+1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + ext v22.16b, v22.16b, v6.16b, #8 // GHASH block 4k+2 - mid aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v22.16b, v22.16b, v6.16b // GHASH block 4k+2 - mid aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull2 v22.1q, v22.2d, v16.2d // GHASH block 4k+2 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + mov d23, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v23.8b, v23.8b, v7.8b // GHASH block 4k+3 - mid aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - ldp x19, x20, [x0, #16] // AES block 4k+5 - load plaintext - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - mov d4, v7.d[1] // GHASH block 4k+3 - mid + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + pmull v23.1q, v23.1d, v16.1d // GHASH block 4k+3 - mid aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v10.16b, v10.16b, v22.16b + eor v10.16b, v10.16b, v23.16b // GHASH block 4k+2/3 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v22.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v21.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v22.16b, v22.16b, v21.16b // GHASH block 4k+3 - high + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull2 v23.1q, v5.2d, v14.2d // GHASH block 4k+1 - high aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor x19, x19, x13 // AES block 4k+5 - round N low - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + pmull v21.1q, v6.1d, v13.1d // GHASH block 4k+2 - low aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - eor x21, x21, x13 // AES block 4k+6 - round N low + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull v20.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + eor v9.16b, v9.16b, v22.16b + eor v9.16b, v9.16b, v23.16b // GHASH block 4k/1/2/3 - high + pmull v22.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + ldp q6, q7, [x0, #32] + ldp q4, q5, [x0], #64 + eor v20.16b, v20.16b, v21.16b // GHASH block 4k+1 - low + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + eor v11.16b, v11.16b, v22.16b + eor v11.16b, v11.16b, v20.16b // GHASH block 4k/1/2/3 - low + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + eor v10.16b, v10.16b, v9.16b + eor v10.16b, v10.16b, v11.16b // MODULO - karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v20.1q, v9.1d, v8.1d // MODULO - top 64b align with mid aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - movi v8.8b, #0xc2 - pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - cmp x17, #12 // setup flags for AES-128/192/256 check - fmov d5, x19 // AES block 4k+5 - mov low - ldp x6, x7, [x0, #0] // AES block 4k+4 - load plaintext - b.lt Lenc_main_loop_continue // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_main_loop_continue // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - b.eq Lenc_main_loop_continue // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Lenc_main_loop_continue // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 Lenc_main_loop_continue: - shl d8, d8, #56 // mod_constant - eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid - add w12, w12, #1 // CTR block 4k+3 - eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - add x0, x0, #64 // AES input_ptr update - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - rev w9, w12 // CTR block 4k+8 - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor x6, x6, x13 // AES block 4k+4 - round N low - eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up - eor x7, x7, x14 // AES block 4k+4 - round N high - fmov d4, x6 // AES block 4k+4 - mov low - orr x9, x11, x9, lsl #32 // CTR block 4k+8 - eor v7.16b, v9.16b, v7.16b // MODULO - fold into mid - eor x20, x20, x14 // AES block 4k+5 - round N high - eor x24, x24, x14 // AES block 4k+7 - round N high - add w12, w12, #1 // CTR block 4k+8 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - fmov v4.d[1], x7 // AES block 4k+4 - mov high - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - fmov d7, x23 // AES block 4k+7 - mov low - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - fmov v5.d[1], x20 // AES block 4k+5 - mov high - fmov d6, x21 // AES block 4k+6 - mov low - cmp x0, x5 // LOOP CONTROL - fmov v6.d[1], x22 // AES block 4k+6 - mov high - pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result - fmov d0, x10 // CTR block 4k+8 - fmov v0.d[1], x9 // CTR block 4k+8 - rev w9, w12 // CTR block 4k+9 - add w12, w12, #1 // CTR block 4k+9 - eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result - fmov d1, x10 // CTR block 4k+9 - orr x9, x11, x9, lsl #32 // CTR block 4k+9 - fmov v1.d[1], x9 // CTR block 4k+9 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - rev w9, w12 // CTR block 4k+10 - st1 { v4.16b}, [x2], #16 // AES block 4k+4 - store result - orr x9, x11, x9, lsl #32 // CTR block 4k+10 - eor v11.16b, v11.16b, v9.16b // MODULO - fold into low - fmov v7.d[1], x24 // AES block 4k+7 - mov high - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - st1 { v5.16b}, [x2], #16 // AES block 4k+5 - store result - add w12, w12, #1 // CTR block 4k+10 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result - fmov d2, x10 // CTR block 4k+10 - st1 { v6.16b}, [x2], #16 // AES block 4k+6 - store result - fmov v2.d[1], x9 // CTR block 4k+10 - rev w9, w12 // CTR block 4k+11 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - orr x9, x11, x9, lsl #32 // CTR block 4k+11 - eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result - st1 { v7.16b}, [x2], #16 // AES block 4k+7 - store result + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v20.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v20.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v9.16b, v11.16b + eor v11.16b, v11.16b, v20.16b // MODULO - fold into low + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor v4.16b, v4.16b, v30.16b + eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor v5.16b, v5.16b, v30.16b + eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor v6.16b, v6.16b, v30.16b + eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor v7.16b, v7.16b, v30.16b + eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // We used these registers as temporaries above so reload the RKs. + ldp q20, q21, [x8, #32] // load rk2, rk3 + ldp q22, q23, [x8, #64] // load rk4, rk5 + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + rev w20, w12 + str w20, [sp, #172] + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] + add w12, w12, #4 + cmp x0, x5 // LOOP CONTROL b.lt Lenc_main_loop - Lenc_prepretail: // PREPRETAIL aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free) + aesmc v1.16b, v1.16b // AES block 1 - round 0 + rev64 v6.16b, v6.16b // GHASH block 2 aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - fmov d3, x10 // CTR block 4k+3 + aesmc v2.16b, v2.16b // AES block 2 - round 0 aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free) - fmov v3.d[1], x9 // CTR block 4k+3 - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aesmc v0.16b, v0.16b // AES block 0 - round 0 + rev64 v4.16b, v4.16b // GHASH block 0 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aesmc v2.16b, v2.16b // AES block 2 - round 1 aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free) + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v5.16b, v5.16b // GHASH block 1 aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + aesmc v2.16b, v2.16b // AES block 2 - round 2 aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - mov d10, v17.d[1] // GHASH block 4k - mid + aesmc v3.16b, v3.16b // AES block 3 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid Karatsuba key aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - mov d8, v4.d[1] // GHASH block 4k - mid - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + aesmc v2.16b, v2.16b // AES block 2 - round 3 aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + aesmc v0.16b, v0.16b // AES block 0 - round 2 aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + aesmc v3.16b, v3.16b // AES block 3 - round 1 aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aesmc v1.16b, v1.16b // AES block 1 - round 3 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - mov d4, v5.d[1] // GHASH block 4k+1 - mid + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + mov d4, v5.d[1] // GHASH block 1 - mid aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low + aesmc v0.16b, v0.16b // AES block 0 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - mov d8, v6.d[1] // GHASH block 4k+2 - mid + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + mov d8, v6.d[1] // GHASH block 2 - mid aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free) + aesmc v0.16b, v0.16b // AES block 0 - round 4 + rev64 v7.16b, v7.16b // GHASH block 3 aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - add w12, w12, #1 // CTR block 4k+3 - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aesmc v3.16b, v3.16b // AES block 3 - round 5 aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aesmc v2.16b, v2.16b // AES block 2 - round 4 + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + ins v8.d[1], v8.d[0] // GHASH block 2 - mid aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - mov d4, v7.d[1] // GHASH block 4k+3 - mid + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v9.16b, v9.16b, v4.16b // GHASH block 2 - high + mov d4, v7.d[1] // GHASH block 3 - mid aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + eor v4.8b, v4.8b, v7.8b // GHASH block 3 - mid + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid + aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull v4.1q, v4.1d, v16.1d // GHASH block 3 - mid + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + aesmc v0.16b, v0.16b // AES block 0 - round 5 aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + aesmc v1.16b, v1.16b // AES block 1 - round 6 aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + aesmc v2.16b, v2.16b // AES block 2 - round 6 aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - movi v8.8b, #0xc2 + aesmc v0.16b, v0.16b // AES block 0 - round 6 aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + aesmc v3.16b, v3.16b // AES block 3 - round 6 aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high + aesmc v1.16b, v1.16b // AES block 1 - round 7 + eor v9.16b, v9.16b, v5.16b // GHASH block 3 - high aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + aesmc v0.16b, v0.16b // AES block 0 - round 7 aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - shl d8, d8, #56 // mod_constant + aesmc v3.16b, v3.16b // AES block 3 - round 7 + ldr d8, [sp, #128] aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid - pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aesmc v1.16b, v1.16b // AES block 1 - round 8 + eor v10.16b, v10.16b, v4.16b // GHASH block 3 - mid + pmull v6.1q, v7.1d, v12.1d // GHASH block 3 - low aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - cmp x17, #12 // setup flags for AES-128/192/256 check + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low + aesmc v0.16b, v0.16b // AES block 0 - round 8 + eor v11.16b, v11.16b, v6.16b // GHASH block 3 - low aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v10.16b, v10.16b, v9.16b // karatsuba tidy up + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v10.16b, v10.16b, v9.16b // karatsuba tidy up aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + aesmc v2.16b, v2.16b // AES block 2 - round 8 pmull v4.1q, v9.1d, v8.1d ext v9.16b, v9.16b, v9.16b, #8 eor v10.16b, v10.16b, v11.16b - b.lt Lenc_finish_prepretail // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + b.lt Lenc_finish_prepretail // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - b.eq Lenc_finish_prepretail // branch if AES-192 - - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Lenc_finish_prepretail // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 Lenc_finish_prepretail: + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 eor v10.16b, v10.16b, v4.16b eor v10.16b, v10.16b, v9.16b pmull v4.1q, v10.1d, v8.1d ext v10.16b, v10.16b, v10.16b, #8 - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 eor v11.16b, v11.16b, v4.16b - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 eor v11.16b, v11.16b, v10.16b - -Lenc_tail: // TAIL - ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag - sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process - ldp x6, x7, [x0], #16 // AES block 4k+4 - load plaintext - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high +Lenc_tail: // TAIL: Process remaining 0 to 3 blocks + ext v8.16b, v11.16b, v11.16b, #8 // Save current GHASH state for partial tag feed-in + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ldp x6, x7, [x0], #16 // AES block 0 - load plaintext + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high cmp x5, #48 - fmov d4, x6 // AES block 4k+4 - mov low - fmov v4.d[1], x7 // AES block 4k+4 - mov high - eor v5.16b, v4.16b, v0.16b // AES block 4k+4 - result + fmov d4, x6 // AES block 0 - mov low + fmov v4.d[1], x7 // AES block 0 - mov high + eor v5.16b, v4.16b, v0.16b // AES block 0 - result b.gt Lenc_blocks_more_than_3 cmp x5, #32 mov v3.16b, v2.16b movi v11.8b, #0 movi v9.8b, #0 - sub w12, w12, #1 mov v2.16b, v1.16b movi v10.8b, #0 b.gt Lenc_blocks_more_than_2 mov v3.16b, v1.16b - sub w12, w12, #1 cmp x5, #16 b.gt Lenc_blocks_more_than_1 - sub w12, w12, #1 b Lenc_blocks_less_than_1 Lenc_blocks_more_than_3: // blocks left > 3 - st1 { v5.16b}, [x2], #16 // AES final-3 block - store result - ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high - rev64 v4.16b, v5.16b // GHASH final-3 block - eor x6, x6, x13 // AES final-2 block - round N low - eor v4.16b, v4.16b, v8.16b // feed in partial tag - eor x7, x7, x14 // AES final-2 block - round N high - mov d22, v4.d[1] // GHASH final-3 block - mid - fmov d5, x6 // AES final-2 block - mov low - fmov v5.d[1], x7 // AES final-2 block - mov high - eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid - movi v8.8b, #0 // suppress further partial tag feed in - mov d10, v17.d[1] // GHASH final-3 block - mid - pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low - pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high - pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid - eor v5.16b, v5.16b, v1.16b // AES final-2 block - result + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-3 block + eor x6, x6, x13 // AES final-2 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor x7, x7, x14 // AES final-2 block - round N high + mov d22, v4.d[1] // GHASH final-3 block - mid + fmov d5, x6 // AES final-2 block - mov low + fmov v5.d[1], x7 // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + mov d10, v17.d[1] // GHASH final-3 block - mid + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor v5.16b, v5.16b, v1.16b // AES final-2 block - result Lenc_blocks_more_than_2: // blocks left > 2 - st1 { v5.16b}, [x2], #16 // AES final-2 block - store result - ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high - rev64 v4.16b, v5.16b // GHASH final-2 block - eor x6, x6, x13 // AES final-1 block - round N low - eor v4.16b, v4.16b, v8.16b // feed in partial tag - fmov d5, x6 // AES final-1 block - mov low - eor x7, x7, x14 // AES final-1 block - round N high - fmov v5.d[1], x7 // AES final-1 block - mov high - movi v8.8b, #0 // suppress further partial tag feed in - pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high - mov d22, v4.d[1] // GHASH final-2 block - mid - pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low - eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid - eor v5.16b, v5.16b, v2.16b // AES final-1 block - result - eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high - pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low - eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-2 block + eor x6, x6, x13 // AES final-1 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + fmov d5, x6 // AES final-1 block - mov low + eor x7, x7, x14 // AES final-1 block - round N high + fmov v5.d[1], x7 // AES final-1 block - mov high + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + eor v5.16b, v5.16b, v2.16b // AES final-1 block - result + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid Lenc_blocks_more_than_1: // blocks left > 1 - st1 { v5.16b}, [x2], #16 // AES final-1 block - store result - rev64 v4.16b, v5.16b // GHASH final-1 block - ldp x6, x7, [x0], #16 // AES final block - load input low & high - eor v4.16b, v4.16b, v8.16b // feed in partial tag - movi v8.8b, #0 // suppress further partial tag feed in - eor x6, x6, x13 // AES final block - round N low - mov d22, v4.d[1] // GHASH final-1 block - mid - pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high - eor x7, x7, x14 // AES final block - round N high - eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high - ins v22.d[1], v22.d[0] // GHASH final-1 block - mid - fmov d5, x6 // AES final block - mov low - fmov v5.d[1], x7 // AES final block - mov high - pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid - pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low - eor v5.16b, v5.16b, v3.16b // AES final block - result - eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low -Lenc_blocks_less_than_1: // blocks left <= 1 - and x1, x1, #127 // bit_length %= 128 - mvn x13, xzr // rkN_l = 0xffffffffffffffff - sub x1, x1, #128 // bit_length -= 128 - neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) - ld1 { v18.16b}, [x2] // load existing bytes where the possibly partial last block is to be stored - mvn x14, xzr // rkN_h = 0xffffffffffffffff - and x1, x1, #127 // bit_length %= 128 - lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + st1 { v5.16b}, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block: Byte Swap CT + ldp x6, x7, [x0], #16 // AES final block - load plaintext + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + movi v8.8b, #0 // Clear for next block + eor x6, x6, x13 // AES final block - round N low + mov d22, v4.d[1] // GHASH final-1 block - mid + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor x7, x7, x14 // AES final block - round N high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + fmov d5, x6 // AES final block - mov low + fmov v5.d[1], x7 // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + eor v5.16b, v5.16b, v3.16b // AES final block - result + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low +Lenc_blocks_less_than_1: // Last partial block handling + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x13, xzr // Mask for low 64 bits + sub x1, x1, #128 // + neg x1, x1 // Valid bits in the last block (1-128) + ldr q18, [x2] // Load destination for merging + mvn x14, xzr // Mask for high 64 bits + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block cmp x1, #64 csel x6, x13, x14, lt csel x7, x14, xzr, lt - fmov d0, x6 // ctr0b is mask for last block + fmov d0, x6 // ctr0d is mask for last block fmov v0.d[1], x7 - and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits - rev64 v4.16b, v5.16b // GHASH final block - eor v4.16b, v4.16b, v8.16b // feed in partial tag - bif v5.16b, v18.16b, v0.16b // insert existing bytes in top end of result before storing - pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high - mov d8, v4.d[1] // GHASH final block - mid - rev w9, w12 - pmull v21.1q, v4.1d, v12.1d // GHASH final block - low - eor v9.16b, v9.16b, v20.16b // GHASH final block - high - eor v8.8b, v8.8b, v4.8b // GHASH final block - mid - pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid - eor v11.16b, v11.16b, v21.16b // GHASH final block - low - eor v10.16b, v10.16b, v8.16b // GHASH final block - mid - movi v8.8b, #0xc2 - eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - shl d8, d8, #56 // mod_constant - eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - str w9, [x16, #12] // store the updated counter - st1 { v5.16b}, [x2] // store all 16B - eor v11.16b, v11.16b, v9.16b // MODULO - fold into low - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + and v5.16b, v5.16b, v0.16b // Mask out unused bits of the last CT block + rev64 v4.16b, v5.16b // GHASH final block - byte swap + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + bif v5.16b, v18.16b, v0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + mov d8, v4.d[1] // GHASH final block - mid + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + fmov d8, x21 + eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + st1 { v5.16b}, [x2] // store all 16B + eor v11.16b, v11.16b, v9.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap GHASH result + rev64 v11.16b, v11.16b // Final Tag mov x0, x15 - st1 { v11.16b }, [x3] + st1 { v11.16b }, [x3] // Store final tag ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] ldp d8, d9, [sp, #64] ldp d10, d11, [sp, #80] ldp d12, d13, [sp, #96] ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 + ldp x29, x30, [sp], #224 AARCH64_VALIDATE_LINK_REGISTER ret @@ -786,9 +751,13 @@ .align 4 aes_gcm_dec_kernel: AARCH64_SIGN_LINK_REGISTER - stp x29, x30, [sp, #-128]! + stp x29, x30, [sp, #-224]! mov x29, sp stp x19, x20, [sp, #16] + ld1 { v0.16b}, [x4] + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b mov x16, x4 mov x8, x5 stp x21, x22, [sp, #32] @@ -797,752 +766,705 @@ stp d10, d11, [sp, #80] stp d12, d13, [sp, #96] stp d14, d15, [sp, #112] - ldr w17, [x8, #240] - add x19, x8, x17, lsl #4 // borrow input_l1 for last key - ldp x13, x14, [x19] // load round N keys - ldr q31, [x19, #-16] // load round N-1 keys - lsr x5, x1, #3 // byte_len + ldr w17, [x8, #240] // Load number of AES rounds + add x19, x8, x17, lsl #4 // borrow input_l1 for last key + ldp x13, x14, [x19] // load round N keys + ldr q31, [x19, #-16] // load round N-1 keys + add x4, x0, x1, lsr #3 // end_input_ptr + lsr x5, x1, #3 // byte_len mov x15, x5 - ldp x10, x11, [x16] // ctr96_b64, ctr96_t32 - ldr q26, [x8, #128] // load rk8 - sub x5, x5, #1 // byte_len - 1 - ldr q25, [x8, #112] // load rk7 - and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - add x4, x0, x1, lsr #3 // end_input_ptr - ldr q24, [x8, #96] // load rk6 - lsr x12, x11, #32 - ldr q23, [x8, #80] // load rk5 - orr w11, w11, w11 - ldr q21, [x8, #48] // load rk3 + ldr w9, [x16, #12] // Load scalar 32-bit counter (CTR) + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) add x5, x5, x0 - rev w12, w12 // rev_ctr32 - add w12, w12, #1 // increment rev_ctr32 - fmov d3, x10 // CTR block 3 - rev w9, w12 // CTR block 1 - add w12, w12, #1 // CTR block 1 - fmov d1, x10 // CTR block 1 - orr x9, x11, x9, lsl #32 // CTR block 1 - ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible - fmov v1.d[1], x9 // CTR block 1 - rev w9, w12 // CTR block 2 - add w12, w12, #1 // CTR block 2 - fmov d2, x10 // CTR block 2 - orr x9, x11, x9, lsl #32 // CTR block 2 - fmov v2.d[1], x9 // CTR block 2 - rev w9, w12 // CTR block 3 - orr x9, x11, x9, lsl #32 // CTR block 3 - ldr q18, [x8, #0] // load rk0 - fmov v3.d[1], x9 // CTR block 3 - add w12, w12, #1 // CTR block 3 - ldr q22, [x8, #64] // load rk4 - ldr q19, [x8, #16] // load rk1 + rev w9, w9 // Reverse it once for big-endian incrementing + uxtw x10, w9 // Zero extend reversed w9 into x10 + str q0, [sp, #160] + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + rev w20, w9 + mov v0.s[3], w20 + add w20, w9, #1 + rev w20, w20 + mov v1.s[3], w20 + add w20, w9, #2 + rev w20, w20 + mov v2.s[3], w20 + add w20, w9, #3 + rev w20, w20 + mov v3.s[3], w20 + add w20, w9, #4 + rev w20, w20 + str w20, [sp, #172] + add w20, w9, #5 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #6 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #7 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov x21, #0xc200000000000000 + str x21, [sp, #128] + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load h2, h3 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 + ldr q15, [x6, #80] // load h4 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // h4l | h3l + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load h1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // h2l | h1l + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_first_blocks // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_first_blocks // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Ldec_finish_first_blocks: + ldr q27, [x19] // load rkN + cmp x0, x5 // check if we have <= 4 blocks + eor v17.16b, v17.16b, v9.16b // h4k | h3k + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // h2h | h1h + eor v16.16b, v16.16b, v8.16b // h2k | h1k + b.ge Ldec_tail // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value (v30). This avoids several fmovs. + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + eor v0.16b, v4.16b, v0.16b + eor v0.16b, v0.16b, v27.16b // AES block 0 - result + eor v1.16b, v5.16b, v1.16b + eor v1.16b, v1.16b, v27.16b // AES block 1 - result + eor v2.16b, v6.16b, v2.16b + eor v2.16b, v2.16b, v27.16b // AES block 2 - result + eor v3.16b, v7.16b, v3.16b + eor v3.16b, v3.16b, v27.16b // AES block 3 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 0-3 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 // check if we have <= 4 blocks + b.ge Ldec_prepretail // do prepretail +Ldec_main_loop: // main loop start + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + rev64 v4.16b, v4.16b // GHASH block 4k + rev64 v5.16b, v5.16b // GHASH block 4k+1 + rev64 v6.16b, v6.16b // GHASH block 4k+2 + rev64 v7.16b, v7.16b // GHASH block 4k+3 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // PRE 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d8, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + mov d4, v5.d[1] // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v5.16b // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + mov d8, v6.d[1] // GHASH block 4k+2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid + ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v28.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + mov d6, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull v27.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + eor v10.16b, v10.16b, v4.16b + eor v10.16b, v10.16b, v8.16b // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid + eor v9.16b, v9.16b, v28.16b + eor v9.16b, v9.16b, v5.16b // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid + ldr d8, [sp, #128] + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + eor v10.16b, v10.16b, v6.16b + eor v10.16b, v10.16b, v7.16b // GHASH block 4k+3 - mid & MODULO - fold into mid + eor v11.16b, v11.16b, v27.16b // GHASH block 4k+3 - low + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v6.16b + eor v10.16b, v10.16b, v9.16b // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_main_loop_continue // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Ldec_main_loop_continue // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +Ldec_main_loop_continue: + ldr q27, [x19] // load rkN + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor v0.16b, v4.16b, v0.16b + eor v0.16b, v0.16b, v27.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor v1.16b, v5.16b, v1.16b + eor v1.16b, v1.16b, v27.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor v2.16b, v6.16b, v2.16b + eor v2.16b, v2.16b, v27.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor v3.16b, v7.16b, v3.16b + eor v3.16b, v3.16b, v27.16b // AES block 4k+7 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 + b.lt Ldec_main_loop +Ldec_prepretail: // PREPRETAIL + rev64 v4.16b, v4.16b // GHASH block 0 + rev64 v5.16b, v5.16b // GHASH block 1 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 aese v0.16b, v18.16b aesmc v0.16b, v0.16b // AES block 0 - round 0 - ldr q14, [x6, #48] // load h3l | h3h - ext v14.16b, v14.16b, v14.16b, #8 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 3 - round 0 - ldr q15, [x6, #80] // load h4l | h4h - ext v15.16b, v15.16b, v15.16b, #8 aese v1.16b, v18.16b aesmc v1.16b, v1.16b // AES block 1 - round 0 - ldr q13, [x6, #32] // load h2l | h2h - ext v13.16b, v13.16b, v13.16b, #8 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 2 - round 0 - ldr q20, [x8, #32] // load rk2 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v6.16b, v6.16b // GHASH block 2 aese v1.16b, v19.16b aesmc v1.16b, v1.16b // AES block 1 - round 1 - ld1 { v11.16b}, [x3] - ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high aese v2.16b, v19.16b aesmc v2.16b, v2.16b // AES block 2 - round 1 - ldr q27, [x8, #144] // load rk9 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low aese v3.16b, v19.16b aesmc v3.16b, v3.16b // AES block 3 - round 1 - ldr q30, [x8, #192] // load rk12 + mov d4, v5.d[1] // GHASH block 1 - mid aese v0.16b, v20.16b aesmc v0.16b, v0.16b // AES block 0 - round 2 - ldr q12, [x6] // load h1l | h1h - ext v12.16b, v12.16b, v12.16b, #8 - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 2 - round 2 - ldr q28, [x8, #160] // load rk10 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 3 - round 2 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 0 - round 3 aese v1.16b, v20.16b aesmc v1.16b, v1.16b // AES block 1 - round 2 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + mov d8, v6.d[1] // GHASH block 2 - mid + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low aese v0.16b, v22.16b aesmc v0.16b, v0.16b // AES block 0 - round 4 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + ins v8.d[1], v8.d[0] // GHASH block 2 - mid aese v2.16b, v21.16b aesmc v2.16b, v2.16b // AES block 2 - round 3 aese v1.16b, v21.16b aesmc v1.16b, v1.16b // AES block 1 - round 3 - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 3 - round 4 + eor v9.16b, v9.16b, v4.16b + eor v9.16b, v9.16b, v5.16b // GHASH block 2 - high & GHASH block 3 - high + pmull v4.1q, v7.1d, v12.1d // GHASH block 3 - low aese v2.16b, v22.16b aesmc v2.16b, v2.16b // AES block 2 - round 4 + mov d6, v7.d[1] // GHASH block 3 - mid aese v1.16b, v22.16b aesmc v1.16b, v1.16b // AES block 1 - round 4 - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 3 - round 5 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 0 - round 5 - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid aese v2.16b, v23.16b aesmc v2.16b, v2.16b // AES block 2 - round 5 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 0 - round 6 + eor v6.8b, v6.8b, v7.8b // GHASH block 3 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 aese v3.16b, v24.16b aesmc v3.16b, v3.16b // AES block 3 - round 6 - cmp x17, #12 // setup flags for AES-128/192/256 check - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid aese v2.16b, v24.16b aesmc v2.16b, v2.16b // AES block 2 - round 6 - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 0 - round 7 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + movi v8.8b, #0xc2 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v11.16b, v11.16b, v4.16b // GHASH block 3 - low + pmull v6.1q, v6.1d, v16.1d // GHASH block 3 - mid aese v3.16b, v25.16b aesmc v3.16b, v3.16b // AES block 3 - round 7 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 0 - round 8 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + eor v10.16b, v10.16b, v6.16b // GHASH block 3 - mid aese v3.16b, v26.16b aesmc v3.16b, v3.16b // AES block 3 - round 8 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up aese v1.16b, v26.16b aesmc v1.16b, v1.16b // AES block 1 - round 8 - ldr q29, [x8, #176] // load rk11 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + shl d8, d8, #56 // mod_constant aese v2.16b, v26.16b aesmc v2.16b, v2.16b // AES block 2 - round 8 - b.lt Ldec_finish_first_blocks // branch if AES-128 - + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_prepretail // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 0 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 9 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 1 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 3 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 9 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 2 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 0 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 10 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 1 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 3 - round 10 + aesmc v1.16b, v1.16b // AES block 1 - round 10 aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 2 - round 10 - b.eq Ldec_finish_first_blocks // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 0 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 3 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 1 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 2 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 1 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 0 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 2 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 3 - round 12 - -Ldec_finish_first_blocks: - cmp x0, x5 // check if we have <= 4 blocks - trn1 v9.2d, v14.2d, v15.2d // h4h | h3h - trn2 v17.2d, v14.2d, v15.2d // h4l | h3l - trn1 v8.2d, v12.2d, v13.2d // h2h | h1h - trn2 v16.2d, v12.2d, v13.2d // h2l | h1l - eor v17.16b, v17.16b, v9.16b // h4k | h3k - aese v1.16b, v31.16b // AES block 1 - round N-1 - aese v2.16b, v31.16b // AES block 2 - round N-1 - eor v16.16b, v16.16b, v8.16b // h2k | h1k - aese v3.16b, v31.16b // AES block 3 - round N-1 - aese v0.16b, v31.16b // AES block 0 - round N-1 - b.ge Ldec_tail // handle tail - - ldr q4, [x0, #0] // AES block 0 - load ciphertext - ldr q5, [x0, #16] // AES block 1 - load ciphertext - rev w9, w12 // CTR block 4 - eor v0.16b, v4.16b, v0.16b // AES block 0 - result - eor v1.16b, v5.16b, v1.16b // AES block 1 - result - rev64 v5.16b, v5.16b // GHASH block 1 - ldr q7, [x0, #48] // AES block 3 - load ciphertext - mov x7, v0.d[1] // AES block 0 - mov high - mov x6, v0.d[0] // AES block 0 - mov low - rev64 v4.16b, v4.16b // GHASH block 0 - add w12, w12, #1 // CTR block 4 - fmov d0, x10 // CTR block 4 - orr x9, x11, x9, lsl #32 // CTR block 4 - fmov v0.d[1], x9 // CTR block 4 - rev w9, w12 // CTR block 5 - add w12, w12, #1 // CTR block 5 - mov x19, v1.d[0] // AES block 1 - mov low - orr x9, x11, x9, lsl #32 // CTR block 5 - mov x20, v1.d[1] // AES block 1 - mov high - eor x7, x7, x14 // AES block 0 - round N high - eor x6, x6, x13 // AES block 0 - round N low - stp x6, x7, [x2], #16 // AES block 0 - store result - fmov d1, x10 // CTR block 5 - ldr q6, [x0, #32] // AES block 2 - load ciphertext - add x0, x0, #64 // AES input_ptr update - fmov v1.d[1], x9 // CTR block 5 - rev w9, w12 // CTR block 6 - add w12, w12, #1 // CTR block 6 - eor x19, x19, x13 // AES block 1 - round N low - orr x9, x11, x9, lsl #32 // CTR block 6 - eor x20, x20, x14 // AES block 1 - round N high - stp x19, x20, [x2], #16 // AES block 1 - store result - eor v2.16b, v6.16b, v2.16b // AES block 2 - result - cmp x0, x5 // check if we have <= 8 blocks - b.ge Ldec_prepretail // do prepretail - -Ldec_main_loop: // main loop start - mov x21, v2.d[0] // AES block 4k+2 - mov low - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - mov x22, v2.d[1] // AES block 4k+2 - mov high - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d2, x10 // CTR block 4k+6 - fmov v2.d[1], x9 // CTR block 4k+6 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev w9, w12 // CTR block 4k+7 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - mov x24, v3.d[1] // AES block 4k+3 - mov high - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - mov x23, v3.d[0] // AES block 4k+3 - mov low - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - mov d8, v4.d[1] // GHASH block 4k - mid - fmov d3, x10 // CTR block 4k+7 - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - orr x9, x11, x9, lsl #32 // CTR block 4k+7 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - fmov v3.d[1], x9 // CTR block 4k+7 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - eor x22, x22, x14 // AES block 4k+2 - round N high - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - mov d10, v17.d[1] // GHASH block 4k - mid - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - rev64 v6.16b, v6.16b // GHASH block 4k+2 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - eor x21, x21, x13 // AES block 4k+2 - round N low - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - stp x21, x22, [x2], #16 // AES block 4k+2 - store result - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - rev64 v7.16b, v7.16b // GHASH block 4k+3 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - eor x23, x23, x13 // AES block 4k+3 - round N low - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - eor x24, x24, x14 // AES block 4k+3 - round N high - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - add w12, w12, #1 // CTR block 4k+7 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - mov d8, v6.d[1] // GHASH block 4k+2 - mid - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - rev w9, w12 // CTR block 4k+8 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - add w12, w12, #1 // CTR block 4k+8 - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - mov d6, v7.d[1] // GHASH block 4k+3 - mid - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - orr x9, x11, x9, lsl #32 // CTR block 4k+8 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - cmp x17, #12 // setup flags for AES-128/192/256 check - eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid - movi v8.8b, #0xc2 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - shl d8, d8, #56 // mod_constant - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - b.lt Ldec_main_loop_continue // branch if AES-128 - + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_prepretail // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v0.16b, v0.16b // AES block 0 - round 11 aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 - b.eq Ldec_main_loop_continue // branch if AES-192 - - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - -Ldec_main_loop_continue: - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - ldr q4, [x0, #0] // AES block 4k+4 - load ciphertext - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - ldr q5, [x0, #16] // AES block 4k+5 - load ciphertext - eor v0.16b, v4.16b, v0.16b // AES block 4k+4 - result - stp x23, x24, [x2], #16 // AES block 4k+3 - store result - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - ldr q7, [x0, #48] // AES block 4k+7 - load ciphertext - ldr q6, [x0, #32] // AES block 4k+6 - load ciphertext - mov x7, v0.d[1] // AES block 4k+4 - mov high - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - add x0, x0, #64 // AES input_ptr update - mov x6, v0.d[0] // AES block 4k+4 - mov low - fmov d0, x10 // CTR block 4k+8 - fmov v0.d[1], x9 // CTR block 4k+8 - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor v1.16b, v5.16b, v1.16b // AES block 4k+5 - result - rev w9, w12 // CTR block 4k+9 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - orr x9, x11, x9, lsl #32 // CTR block 4k+9 - cmp x0, x5 // LOOP CONTROL - add w12, w12, #1 // CTR block 4k+9 - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high - mov x20, v1.d[1] // AES block 4k+5 - mov high - eor v2.16b, v6.16b, v2.16b // AES block 4k+6 - result - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low - mov x19, v1.d[0] // AES block 4k+5 - mov low - fmov d1, x10 // CTR block 4k+9 - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - fmov v1.d[1], x9 // CTR block 4k+9 - rev w9, w12 // CTR block 4k+10 - add w12, w12, #1 // CTR block 4k+10 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - orr x9, x11, x9, lsl #32 // CTR block 4k+10 - rev64 v5.16b, v5.16b // GHASH block 4k+5 - eor x20, x20, x14 // AES block 4k+5 - round N high - stp x6, x7, [x2], #16 // AES block 4k+4 - store result - eor x19, x19, x13 // AES block 4k+5 - round N low - stp x19, x20, [x2], #16 // AES block 4k+5 - store result - rev64 v4.16b, v4.16b // GHASH block 4k+4 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - b.lt Ldec_main_loop - -Ldec_prepretail: // PREPRETAIL - ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 - mov x21, v2.d[0] // AES block 4k+2 - mov low - eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 - mov x22, v2.d[1] // AES block 4k+2 - mov high - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 - fmov d2, x10 // CTR block 4k+6 - fmov v2.d[1], x9 // CTR block 4k+6 - rev w9, w12 // CTR block 4k+7 - eor v4.16b, v4.16b, v11.16b // PRE 1 - rev64 v6.16b, v6.16b // GHASH block 4k+2 - orr x9, x11, x9, lsl #32 // CTR block 4k+7 - mov x23, v3.d[0] // AES block 4k+3 - mov low - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 - mov x24, v3.d[1] // AES block 4k+3 - mov high - pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low - mov d8, v4.d[1] // GHASH block 4k - mid - fmov d3, x10 // CTR block 4k+7 - pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high - fmov v3.d[1], x9 // CTR block 4k+7 - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 - mov d10, v17.d[1] // GHASH block 4k - mid - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 - eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid - pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 - rev64 v7.16b, v7.16b // GHASH block 4k+3 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 - pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high - pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 - mov d4, v5.d[1] // GHASH block 4k+1 - mid - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 - eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 - mov d8, v6.d[1] // GHASH block 4k+2 - mid - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 - eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid - pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 - eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid - pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 - eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 - pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high - eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid - pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 - ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 - eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high - pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 - mov d6, v7.d[1] // GHASH block 4k+3 - mid - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 - pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 - eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 - eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 - movi v8.8b, #0xc2 - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 - eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low - pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 - cmp x17, #12 // setup flags for AES-128/192/256 check - eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 - eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 - shl d8, d8, #56 // mod_constant - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 - b.lt Ldec_finish_prepretail // branch if AES-128 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aesmc v1.16b, v1.16b // AES block 1 - round 11 aese v2.16b, v27.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aesmc v2.16b, v2.16b // AES block 2 - round 11 aese v3.16b, v27.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + aesmc v3.16b, v3.16b // AES block 3 - round 11 aese v0.16b, v28.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aesmc v0.16b, v0.16b // AES block 0 - round 12 aese v1.16b, v28.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 - b.eq Ldec_finish_prepretail // branch if AES-192 - - aese v2.16b, v29.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 - aese v0.16b, v29.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 - aese v1.16b, v29.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 - aese v2.16b, v30.16b - aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 - aese v3.16b, v29.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 - aese v1.16b, v30.16b - aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 - aese v0.16b, v30.16b - aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 - aese v3.16b, v30.16b - aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 - + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 Ldec_finish_prepretail: - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor x22, x22, x14 // AES block 4k+2 - round N high - eor x23, x23, x13 // AES block 4k+3 - round N low - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - add w12, w12, #1 // CTR block 4k+7 - eor x21, x21, x13 // AES block 4k+2 - round N low - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - eor x24, x24, x14 // AES block 4k+3 - round N high - stp x21, x22, [x2], #16 // AES block 4k+2 - store result - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - stp x23, x24, [x2], #16 // AES block 4k+3 - store result - - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low - aese v1.16b, v31.16b // AES block 4k+5 - round N-1 - aese v0.16b, v31.16b // AES block 4k+4 - round N-1 - aese v3.16b, v31.16b // AES block 4k+7 - round N-1 - aese v2.16b, v31.16b // AES block 4k+6 - round N-1 - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low - + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 Ldec_tail: // TAIL - sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process - ld1 { v5.16b}, [x0], #16 // AES block 4k+4 - load ciphertext - eor v0.16b, v5.16b, v0.16b // AES block 4k+4 - result - mov x6, v0.d[0] // AES block 4k+4 - mov low - mov x7, v0.d[1] // AES block 4k+4 - mov high - ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ld1 { v5.16b}, [x0], #16 // AES block 0 - load ciphertext + eor v0.16b, v5.16b, v0.16b // AES block 0 - result + mov x6, v0.d[0] // AES block 0 - mov low + mov x7, v0.d[1] // AES block 0 - mov high + ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high cmp x5, #48 - eor x6, x6, x13 // AES block 4k+4 - round N low - eor x7, x7, x14 // AES block 4k+4 - round N high b.gt Ldec_blocks_more_than_3 - sub w12, w12, #1 mov v3.16b, v2.16b movi v10.8b, #0 movi v11.8b, #0 - cmp x5, #32 movi v9.8b, #0 mov v2.16b, v1.16b + cmp x5, #32 b.gt Ldec_blocks_more_than_2 - sub w12, w12, #1 mov v3.16b, v1.16b cmp x5, #16 b.gt Ldec_blocks_more_than_1 - sub w12, w12, #1 b Ldec_blocks_less_than_1 Ldec_blocks_more_than_3: // blocks left > 3 - rev64 v4.16b, v5.16b // GHASH final-3 block - ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext - stp x6, x7, [x2], #16 // AES final-3 block - store result - mov d10, v17.d[1] // GHASH final-3 block - mid - eor v4.16b, v4.16b, v8.16b // feed in partial tag - eor v0.16b, v5.16b, v1.16b // AES final-2 block - result - mov d22, v4.d[1] // GHASH final-3 block - mid - mov x6, v0.d[0] // AES final-2 block - mov low - mov x7, v0.d[1] // AES final-2 block - mov high - eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid - movi v8.8b, #0 // suppress further partial tag feed in - pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high - pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid - eor x6, x6, x13 // AES final-2 block - round N low - pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low - eor x7, x7, x14 // AES final-2 block - round N high + rev64 v4.16b, v5.16b // GHASH final-3 block + ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext + stp x6, x7, [x2], #16 // AES final-3 block - store result + mov d10, v17.d[1] // GHASH final-3 block - mid + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor v0.16b, v5.16b, v1.16b // AES final-2 block - result + mov d22, v4.d[1] // GHASH final-3 block - mid + mov x6, v0.d[0] // AES final-2 block - mov low + mov x7, v0.d[1] // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor x6, x6, x13 // AES final-2 block - round N low + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + eor x7, x7, x14 // AES final-2 block - round N high Ldec_blocks_more_than_2: // blocks left > 2 - rev64 v4.16b, v5.16b // GHASH final-2 block - ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext - eor v4.16b, v4.16b, v8.16b // feed in partial tag - stp x6, x7, [x2], #16 // AES final-2 block - store result - eor v0.16b, v5.16b, v2.16b // AES final-1 block - result - mov d22, v4.d[1] // GHASH final-2 block - mid - pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low - pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high - eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid - mov x6, v0.d[0] // AES final-1 block - mov low - mov x7, v0.d[1] // AES final-1 block - mov high - eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low - movi v8.8b, #0 // suppress further partial tag feed in - pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high - eor x6, x6, x13 // AES final-1 block - round N low - eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid - eor x7, x7, x14 // AES final-1 block - round N high + rev64 v4.16b, v5.16b // GHASH final-2 block + ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + stp x6, x7, [x2], #16 // AES final-2 block - store result + eor v0.16b, v5.16b, v2.16b // AES final-1 block - result + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + mov x6, v0.d[0] // AES final-1 block - mov low + mov x7, v0.d[1] // AES final-1 block - mov high + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + movi v8.8b, #0 // suppress further partial tag feed in + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + eor x6, x6, x13 // AES final-1 block - round N low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + eor x7, x7, x14 // AES final-1 block - round N high Ldec_blocks_more_than_1: // blocks left > 1 - stp x6, x7, [x2], #16 // AES final-1 block - store result - rev64 v4.16b, v5.16b // GHASH final-1 block - ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext - eor v4.16b, v4.16b, v8.16b // feed in partial tag - movi v8.8b, #0 // suppress further partial tag feed in - mov d22, v4.d[1] // GHASH final-1 block - mid - eor v0.16b, v5.16b, v3.16b // AES final block - result - pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high - eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid - pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low - mov x6, v0.d[0] // AES final block - mov low - ins v22.d[1], v22.d[0] // GHASH final-1 block - mid - mov x7, v0.d[1] // AES final block - mov high - pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid - eor x6, x6, x13 // AES final block - round N low - eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low - eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high - eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid - eor x7, x7, x14 // AES final block - round N high + stp x6, x7, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block + ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + movi v8.8b, #0 // suppress further partial tag feed in + mov d22, v4.d[1] // GHASH final-1 block - mid + eor v0.16b, v5.16b, v3.16b // AES final block - result + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + mov x6, v0.d[0] // AES final block - mov low + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + mov x7, v0.d[1] // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + eor x6, x6, x13 // AES final block - round N low + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor x7, x7, x14 // AES final block - round N high Ldec_blocks_less_than_1: // blocks left <= 1 - and x1, x1, #127 // bit_length %= 128 - mvn x14, xzr // rkN_h = 0xffffffffffffffff - sub x1, x1, #128 // bit_length -= 128 - mvn x13, xzr // rkN_l = 0xffffffffffffffff - ldp x4, x5, [x2] // load existing bytes we need to not overwrite - neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) - and x1, x1, #127 // bit_length %= 128 - lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x14, xzr // rkN_h = 0xffffffffffffffff + sub x1, x1, #128 // bit_length -= 128 + mvn x13, xzr // rkN_l = 0xffffffffffffffff + ldp x4, x5, [x2] // load existing bytes we need to not overwrite + neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block cmp x1, #64 csel x9, x13, x14, lt csel x10, x14, xzr, lt - fmov d0, x9 // ctr0b is mask for last block + fmov d0, x9 // ctr0b is mask for last block and x6, x6, x9 mov v0.d[1], x10 - bic x4, x4, x9 // mask out low existing bytes - rev w9, w12 - bic x5, x5, x10 // mask out high existing bytes + bic x4, x4, x9 // mask out low existing bytes + bic x5, x5, x10 // mask out high existing bytes orr x6, x6, x4 and x7, x7, x10 orr x7, x7, x5 - and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits - rev64 v4.16b, v5.16b // GHASH final block - eor v4.16b, v4.16b, v8.16b // feed in partial tag - pmull v21.1q, v4.1d, v12.1d // GHASH final block - low - mov d8, v4.d[1] // GHASH final block - mid - eor v8.8b, v8.8b, v4.8b // GHASH final block - mid - pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high - pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid - eor v9.16b, v9.16b, v20.16b // GHASH final block - high - eor v11.16b, v11.16b, v21.16b // GHASH final block - low - eor v10.16b, v10.16b, v8.16b // GHASH final block - mid - movi v8.8b, #0xc2 - eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up - shl d8, d8, #56 // mod_constant - eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up - pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid - ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment - eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid - eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid - pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low - ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment - eor v11.16b, v11.16b, v8.16b // MODULO - fold into low + and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits + rev64 v4.16b, v5.16b // GHASH final block + eor v4.16b, v4.16b, v8.16b // feed in partial tag + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + mov d8, v4.d[1] // GHASH final block - mid + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + ldr d8, [sp, #128] + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b // MODULO - fold into low stp x6, x7, [x2] - str w9, [x16, #12] // store the updated counter - eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low ext v11.16b, v11.16b, v11.16b, #8 - rev64 v11.16b, v11.16b + rev64 v11.16b, v11.16b // Final Tag mov x0, x15 - st1 { v11.16b }, [x3] + st1 { v11.16b }, [x3] // Store final tag ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x23, x24, [sp, #48] @@ -1550,9 +1472,1444 @@ ldp d10, d11, [sp, #80] ldp d12, d13, [sp, #96] ldp d14, d15, [sp, #112] - ldp x29, x30, [sp], #128 + ldp x29, x30, [sp], #224 AARCH64_VALIDATE_LINK_REGISTER ret -#endif + +.globl aes_gcm_enc_kernel_eor3 + +.def aes_gcm_enc_kernel_eor3 + .type 32 +.endef +.align 4 +aes_gcm_enc_kernel_eor3: + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + ld1 { v0.16b}, [x4] // Load initial counter block + stp x19, x20, [sp, #16] + mov v1.16b, v0.16b // Initialize ctr1-3 from ctr0 + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 // Pointer to counter block in memory + mov x8, x5 // Pointer to AES key schedule context + stp x21, x22, [sp, #32] + // [sp, #48] is unused but allocated to align the stack layout with aes_gcm_dec_kernel_eor3 + stp d8, d9, [sp, #64] // Save Neon registers + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr w17, [x8, #240] // Load number of AES rounds + add x7, x8, x17, lsl #4 // Calculate pointer to the last round key + ldp x13, x14, [x7] // load round N key (for final XOR) + ldr q31, [x7, #-16] // load round N-1 key + add x4, x0, x1, lsr #3 // Calculate end of input + lsr x5, x1, #3 // Total byte length + mov x15, x5 + ldr w12, [x16, #12] // Load counter's low 32 bits + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // Align main loop end to a multiple of 64 bytes + add x5, x5, x0 + rev w12, w12 // Reverse for big-endian increment + uxtw x10, w12 // Zero extend reversed w12 into x10 for final counter update + // Pre-compute this value instead of using two instructions to reconstruct it every iteration + mov x21, #0xc200000000000000 // GHASH reduction constant + str x21, [sp, #128] + // We maintain four copies of ctr values on the stack. Each loop iteration we + // store the updated ctr value to the last four bytes (e.g., 160 + 12). + // We then load the four values. This avoids a singificant number of + // expensive GPR->NEON and NEON->NEON moves. To avoid LDST forwarding we + // calculate and store the values one iteration ahead so they have time to + // drain before we load them. + str q0, [sp, #160] // Store base counter for block 0-3 + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + // Since we need the values right away don't go through the stack this first + // time. Manually insert the incremented big-endian counter values. + rev w20, w12 + mov v0.s[3], w20 // ctr0 + 0 + add w20, w12, #1 + rev w20, w20 + mov v1.s[3], w20 // ctr0 + 1 + add w20, w12, #2 + rev w20, w20 + mov v2.s[3], w20 // ctr0 + 2 + add w20, w12, #3 + rev w20, w20 + mov v3.s[3], w20 // ctr0 + 3 + // Calculate the ctr values for the *next* (not current) group of four + // blocks. Store the incremented parts to the stack. + add w20, w12, #4 + rev w20, w20 + str w20, [sp, #172] // ctr0 + 4 for next iter + add w20, w12, #5 + rev w20, w20 + str w20, [sp, #188] // ctr0 + 5 for next iter + add w20, w12, #6 + rev w20, w20 + str w20, [sp, #204] // ctr0 + 6 for next iter + add w20, w12, #7 + rev w20, w20 + str w20, [sp, #220] // ctr0 + 7 for next iter + add w12, w12, #8 // Advance counter past these two sets + // --- Start AES for first 4 blocks --- + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load H2, H3 (GHASH keys) + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 // Byte swap H3 for GHASH + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 // Byte swap H2 for GHASH + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + ldr q15, [x6, #80] // load H4 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 // Byte swap H4 for GHASH + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] // Load initial GHASH accumulator (T) + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap T for GHASH + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b // Correct byte order within 64-bit lanes + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // Karatsuba key: H4_low | H3_low + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load H1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 // Byte swap H1 for GHASH + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // Karatsuba key: H4_high | H3_high + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // Karatsuba key: H2_low | H1_low + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + ldr q30, [x7] // Preload round N key for final EOR + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_finish_first_blocks_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + b.eq Lenc_finish_first_blocks_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 +Lenc_finish_first_blocks_eor3: + cmp x0, x5 // check if we have <= 4 blocks to process in the tail + eor v17.16b, v17.16b, v9.16b // Karatsuba key: H3^H4 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // Karatsuba key: H2_high | H1_high + eor v16.16b, v16.16b, v8.16b // Karatsuba key: H1^H2 + b.ge Lenc_tail_eor3 // handle tail if no more full 4-block sets + ldp q6, q7, [x0, #32] // AES blocks 2,3 load plaintext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load plaintext + // Compute and store first 4 ciphertext blocks + eor3 v4.16b, v4.16b, v30.16b, v0.16b // AES block 0 - result = PT ^ AES(ctr0) + eor3 v5.16b, v5.16b, v30.16b, v1.16b // AES block 1 - result = PT ^ AES(ctr1) + eor3 v6.16b, v6.16b, v30.16b, v2.16b // AES block 2 - result = PT ^ AES(ctr2) + eor3 v7.16b, v7.16b, v30.16b, v3.16b // AES block 3 - result = PT ^ AES(ctr3) + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 0-3 - store result + // Load counter values for the second iteration from the stack + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // Prepare and store counter values for the third iteration + rev w20, w12 + str w20, [sp, #172] // ctr + 8 + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] // ctr + 9 + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] // ctr + 10 + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] // ctr + 11 + add w12, w12, #4 // Advance counter base + cmp x0, x5 // check if we have <= 4 blocks remaining + b.ge Lenc_prepretail_eor3 // go to prepretail if < 2 full loops left +Lenc_main_loop_eor3: // main loop start (processes 4 blocks per iteration) + // --- AES Pipeline for blocks 4k+4 to 4k+7 --- + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + ldr d8, [sp, #128] // Load GHASH reduction constant + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + // --- GHASH Pipeline (interleaved with AES) for blocks 4k to 4k+3 --- + rev64 v4.16b, v4.16b // GHASH block 4k - Byte swap CT + rev64 v5.16b, v5.16b // GHASH block 4k+1 - Byte swap CT + rev64 v6.16b, v6.16b // GHASH block 4k+2 - Byte swap CT + rev64 v7.16b, v7.16b // GHASH block 4k+3 - Byte swap CT + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // GHASH - prepare acc for XOR + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // GHASH block 4k - Y_i = CT_i ^ Y_{i-1} + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid Karatsuba key + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d20, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v20.8b, v20.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + mov d21, v5.d[1] // GHASH block 4k+1 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + eor v21.8b, v21.8b, v5.8b // GHASH block 4k+1 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + pmull v10.1q, v20.1d, v10.1d // GHASH block 4k - mid + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + pmull v21.1q, v21.1d, v17.1d // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + eor v10.16b, v10.16b, v21.16b // GHASH block 4k+1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + ext v22.16b, v22.16b, v6.16b, #8 // GHASH block 4k+2 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v22.16b, v22.16b, v6.16b // GHASH block 4k+2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull2 v22.1q, v22.2d, v16.2d // GHASH block 4k+2 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + mov d23, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor v23.8b, v23.8b, v7.8b // GHASH block 4k+3 - mid + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + pmull v23.1q, v23.1d, v16.1d // GHASH block 4k+3 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor3 v10.16b, v10.16b, v22.16b, v23.16b // GHASH block 4k+2/3 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v22.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v21.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v22.16b, v22.16b, v21.16b // GHASH block 4k+3 - high + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull2 v23.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + pmull v21.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull v20.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + eor3 v9.16b, v9.16b, v22.16b, v23.16b // GHASH block 4k/1/2/3 - high + pmull v22.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + ldp q6, q7, [x0, #32] + ldp q4, q5, [x0], #64 + eor v20.16b, v20.16b, v21.16b // GHASH block 4k+1 - low + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + eor3 v11.16b, v11.16b, v22.16b, v20.16b // GHASH block 4k/1/2/3 - low + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + eor3 v10.16b, v10.16b, v9.16b, v11.16b // MODULO - karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v20.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Lenc_main_loop_continue_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Lenc_main_loop_continue_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +Lenc_main_loop_continue_eor3: + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v20.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v20.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v9.16b, v11.16b, v20.16b // MODULO - fold into low + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor3 v4.16b, v4.16b, v30.16b, v0.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor3 v5.16b, v5.16b, v30.16b, v1.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor3 v6.16b, v6.16b, v30.16b, v2.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor3 v7.16b, v7.16b, v30.16b, v3.16b // AES block 4k+7 - result + ldp q0, q1, [sp, #160] + ldp q2, q3, [sp, #192] + // We used these registers as temporaries above so reload the RKs. + ldp q20, q21, [x8, #32] // load rk2, rk3 + ldp q22, q23, [x8, #64] // load rk4, rk5 + st1 { v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + rev w20, w12 + str w20, [sp, #172] + add w20, w12, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w12, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w12, #3 + rev w20, w20 + str w20, [sp, #220] + add w12, w12, #4 + cmp x0, x5 // LOOP_eor3 CONTROL + b.lt Lenc_main_loop_eor3 +Lenc_prepretail_eor3: // PREPRETAIL + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + rev64 v6.16b, v6.16b // GHASH block 2 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + rev64 v4.16b, v4.16b // GHASH block 0 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v5.16b, v5.16b // GHASH block 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid Karatsuba key + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + mov d4, v5.d[1] // GHASH block 1 - mid + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + mov d8, v6.d[1] // GHASH block 2 - mid + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + ins v8.d[1], v8.d[0] // GHASH block 2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v9.16b, v9.16b, v4.16b // GHASH block 2 - high + mov d4, v7.d[1] // GHASH block 3 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + eor v4.8b, v4.8b, v7.8b // GHASH block 3 - mid + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + pmull v4.1q, v4.1d, v16.1d // GHASH block 3 - mid + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + eor v9.16b, v9.16b, v5.16b // GHASH block 3 - high + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + ldr d8, [sp, #128] + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + eor v10.16b, v10.16b, v4.16b // GHASH block 3 - mid + pmull v6.1q, v7.1d, v12.1d // GHASH block 3 - low + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + eor v11.16b, v11.16b, v6.16b // GHASH block 3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v10.16b, v10.16b, v9.16b // karatsuba tidy up + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + pmull v4.1q, v9.1d, v8.1d + ext v9.16b, v9.16b, v9.16b, #8 + eor v10.16b, v10.16b, v11.16b + b.lt Lenc_finish_prepretail_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Lenc_finish_prepretail_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Lenc_finish_prepretail_eor3: + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + eor3 v10.16b, v10.16b, v4.16b, v9.16b + pmull v4.1q, v10.1d, v8.1d + ext v10.16b, v10.16b, v10.16b, #8 + eor3 v11.16b, v11.16b, v4.16b, v10.16b +Lenc_tail_eor3: // TAIL: Process remaining 0 to 3 blocks + ext v8.16b, v11.16b, v11.16b, #8 // Save current GHASH state for partial tag feed-in + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ldp x6, x7, [x0], #16 // AES block 0 - load plaintext + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high + cmp x5, #48 + fmov d4, x6 // AES block 0 - mov low + fmov v4.d[1], x7 // AES block 0 - mov high + eor v5.16b, v4.16b, v0.16b // AES block 0 - result + b.gt Lenc_blocks_more_than_3_eor3 + cmp x5, #32 + mov v3.16b, v2.16b + movi v11.8b, #0 + movi v9.8b, #0 + mov v2.16b, v1.16b + movi v10.8b, #0 + b.gt Lenc_blocks_more_than_2_eor3 + mov v3.16b, v1.16b + cmp x5, #16 + b.gt Lenc_blocks_more_than_1_eor3 + b Lenc_blocks_less_than_1_eor3 +Lenc_blocks_more_than_3_eor3: // blocks left > 3 + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-3 block + eor x6, x6, x13 // AES final-2 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor x7, x7, x14 // AES final-2 block - round N high + mov d22, v4.d[1] // GHASH final-3 block - mid + fmov d5, x6 // AES final-2 block - mov low + fmov v5.d[1], x7 // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + mov d10, v17.d[1] // GHASH final-3 block - mid + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor v5.16b, v5.16b, v1.16b // AES final-2 block - result +Lenc_blocks_more_than_2_eor3: // blocks left > 2 + st1 { v5.16b}, [x2], #16 // AES final-2 block - store result + ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high + rev64 v4.16b, v5.16b // GHASH final-2 block + eor x6, x6, x13 // AES final-1 block - round N low + eor v4.16b, v4.16b, v8.16b // feed in partial tag + fmov d5, x6 // AES final-1 block - mov low + eor x7, x7, x14 // AES final-1 block - round N high + fmov v5.d[1], x7 // AES final-1 block - mov high + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + eor v5.16b, v5.16b, v2.16b // AES final-1 block - result + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid +Lenc_blocks_more_than_1_eor3: // blocks left > 1 + st1 { v5.16b}, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block: Byte Swap CT + ldp x6, x7, [x0], #16 // AES final block - load plaintext + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + movi v8.8b, #0 // Clear for next block + eor x6, x6, x13 // AES final block - round N low + mov d22, v4.d[1] // GHASH final-1 block - mid + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor x7, x7, x14 // AES final block - round N high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + fmov d5, x6 // AES final block - mov low + fmov v5.d[1], x7 // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + eor v5.16b, v5.16b, v3.16b // AES final block - result + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low +Lenc_blocks_less_than_1_eor3: // Last partial block handling + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x13, xzr // Mask for low 64 bits + sub x1, x1, #128 // + neg x1, x1 // Valid bits in the last block (1-128) + ldr q18, [x2] // Load destination for merging + mvn x14, xzr // Mask for high 64 bits + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + cmp x1, #64 + csel x6, x13, x14, lt + csel x7, x14, xzr, lt + fmov d0, x6 // ctr0d is mask for last block + fmov v0.d[1], x7 + and v5.16b, v5.16b, v0.16b // Mask out unused bits of the last CT block + rev64 v4.16b, v5.16b // GHASH final block - byte swap + eor v4.16b, v4.16b, v8.16b // Feed in partial tag + bif v5.16b, v18.16b, v0.16b // Bitwise Insert: merge with existing data at output_ptr + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + mov d8, v4.d[1] // GHASH final block - mid + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + fmov d8, x21 + eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v7.16b, v9.16b // MODULO - fold into mid + pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + st1 { v5.16b}, [x2] // store all 16B + eor3 v11.16b, v11.16b, v9.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 // Byte swap GHASH result + rev64 v11.16b, v11.16b // Final Tag + mov x0, x15 + st1 { v11.16b }, [x3] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret + +.globl aes_gcm_dec_kernel_eor3 + +.def aes_gcm_dec_kernel_eor3 + .type 32 +.endef +.align 4 +aes_gcm_dec_kernel_eor3: + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-224]! + mov x29, sp + stp x19, x20, [sp, #16] + ld1 { v0.16b}, [x4] + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + mov x16, x4 + mov x8, x5 + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp d8, d9, [sp, #64] + stp d10, d11, [sp, #80] + stp d12, d13, [sp, #96] + stp d14, d15, [sp, #112] + ldr w17, [x8, #240] // Load number of AES rounds + add x19, x8, x17, lsl #4 // borrow input_l1 for last key + ldp x13, x14, [x19] // load round N keys + ldr q31, [x19, #-16] // load round N-1 keys + add x4, x0, x1, lsr #3 // end_input_ptr + lsr x5, x1, #3 // byte_len + mov x15, x5 + ldr w9, [x16, #12] // Load scalar 32-bit counter (CTR) + sub x5, x5, #1 // byte_len - 1 + ldr q18, [x8, #0] // load rk0 + and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + add x5, x5, x0 + rev w9, w9 // Reverse it once for big-endian incrementing + uxtw x10, w9 // Zero extend reversed w9 into x10 + str q0, [sp, #160] + str q0, [sp, #176] + str q0, [sp, #192] + str q0, [sp, #208] + rev w20, w9 + mov v0.s[3], w20 + add w20, w9, #1 + rev w20, w20 + mov v1.s[3], w20 + add w20, w9, #2 + rev w20, w20 + mov v2.s[3], w20 + add w20, w9, #3 + rev w20, w20 + mov v3.s[3], w20 + add w20, w9, #4 + rev w20, w20 + str w20, [sp, #172] + add w20, w9, #5 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #6 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #7 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #8 + // Pre-compute this value instead of using two instructions for moving and then shifting in the main loop + mov x21, #0xc200000000000000 + str x21, [sp, #128] + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + ldp q19, q20, [x8, #16] // load rk1, rk2 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + ldp q21, q22, [x8, #48] // load rk3, rk4 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + ldp q23, q24, [x8, #80] // load rk5, rk6 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + ldp q13, q14, [x6, #32] // load h2, h3 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + ldp q25, q26, [x8, #112] // load rk7, rk8 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + ext v14.16b, v14.16b, v14.16b, #8 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + ext v13.16b, v13.16b, v13.16b, #8 + ldr q15, [x6, #80] // load h4 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + ext v15.16b, v15.16b, v15.16b, #8 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + ld1 { v11.16b}, [x3] + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + ext v11.16b, v11.16b, v11.16b, #8 + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + rev64 v11.16b, v11.16b + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + trn2 v17.2d, v14.2d, v15.2d // h4l | h3l + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + ldr q12, [x6] // load h1 + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + ext v12.16b, v12.16b, v12.16b, #8 + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + trn1 v9.2d, v14.2d, v15.2d // h4h | h3h + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + trn2 v16.2d, v12.2d, v13.2d // h2l | h1l + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + cmp x17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_first_blocks_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_first_blocks_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Ldec_finish_first_blocks_eor3: + ldr q27, [x19] // load rkN + cmp x0, x5 // check if we have <= 4 blocks + eor v17.16b, v17.16b, v9.16b // h4k | h3k + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + trn1 v8.2d, v12.2d, v13.2d // h2h | h1h + eor v16.16b, v16.16b, v8.16b // h2k | h1k + b.ge Ldec_tail_eor3 // handle tail + // Setup for AES blocks 0-3 is done purely on NEON side instead of mixing NEON and scalar instructions. + // This is because the final result of the AES block needs to be EORd with the final round key + // value (v30). This avoids several fmovs. + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + eor3 v0.16b, v4.16b, v0.16b, v27.16b // AES block 0 - result + eor3 v1.16b, v5.16b, v1.16b, v27.16b // AES block 1 - result + eor3 v2.16b, v6.16b, v2.16b, v27.16b // AES block 2 - result + eor3 v3.16b, v7.16b, v3.16b, v27.16b // AES block 3 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 0-3 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 // check if we have <= 4 blocks + b.ge Ldec_prepretail_eor3 // do prepretail +Ldec_main_loop_eor3: // main loop start + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 0 + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 0 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 0 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 1 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 1 + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 1 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 1 + rev64 v4.16b, v4.16b // GHASH block 4k + rev64 v5.16b, v5.16b // GHASH block 4k+1 + rev64 v6.16b, v6.16b // GHASH block 4k+2 + rev64 v7.16b, v7.16b // GHASH block 4k+3 + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 2 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 2 + eor v4.16b, v4.16b, v11.16b // PRE 1 + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 2 + pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 2 + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 3 + mov d10, v17.d[1] // GHASH block 4k - mid + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 3 + mov d8, v4.d[1] // GHASH block 4k - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 3 + eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 3 + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 4 + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 4 + eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 4 + mov d4, v5.d[1] // GHASH block 4k+1 - mid + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 4 + pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 5 + pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 5 + eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 5 + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 5 + pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 6 + eor3 v11.16b, v11.16b, v8.16b, v5.16b // GHASH block 4k+1 - low & GHASH block 4k+2 - low + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 6 + mov d8, v6.d[1] // GHASH block 4k+2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 6 + eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid + ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 6 + pmull2 v28.1q, v6.2d, v13.2d // GHASH block 4k+2 - high + mov d6, v7.d[1] // GHASH block 4k+3 - mid + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 7 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 7 + pmull v27.1q, v7.1d, v12.1d // GHASH block 4k+3 - low + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 7 + eor3 v10.16b, v10.16b, v4.16b, v8.16b // GHASH block 4k+1 - mid & GHASH block 4k+2 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 7 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high + eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid + eor3 v9.16b, v9.16b, v28.16b, v5.16b // GHASH block 4k+2 - high & GHASH block 4k+3 - high + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 8 + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 8 + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 8 + pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid + ldr d8, [sp, #128] + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 8 + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + eor3 v10.16b, v10.16b, v6.16b, v7.16b // GHASH block 4k+3 - mid & MODULO - fold into mid + eor v11.16b, v11.16b, v27.16b // GHASH block 4k+3 - low + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v6.16b, v9.16b // MODULO - karatsuba tidy up & MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v11.16b, v8.16b, v10.16b // MODULO - fold into low + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_main_loop_continue_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 10 + b.eq Ldec_main_loop_continue_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 4k+4 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 4k+5 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 4k+6 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 4k+7 - round 12 +Ldec_main_loop_continue_eor3: + ldr q27, [x19] // load rkN + ldp q6, q7, [x0, #32] // AES blocks 2,3 load ciphertext + ldp q4, q5, [x0], #64 // AES blocks 0,1 load ciphertext + aese v0.16b, v31.16b // AES block 4k+4 - round N-1 + eor3 v0.16b, v4.16b, v0.16b, v27.16b // AES block 4k+4 - result + aese v1.16b, v31.16b // AES block 4k+5 - round N-1 + eor3 v1.16b, v5.16b, v1.16b, v27.16b // AES block 4k+5 - result + aese v2.16b, v31.16b // AES block 4k+6 - round N-1 + eor3 v2.16b, v6.16b, v2.16b, v27.16b // AES block 4k+6 - result + aese v3.16b, v31.16b // AES block 4k+7 - round N-1 + eor3 v3.16b, v7.16b, v3.16b, v27.16b // AES block 4k+7 - result + st1 { v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 // AES blocks 4k+4-7 - store result + ldr q0, [sp, #160] + ldr q1, [sp, #176] + ldr q2, [sp, #192] + ldr q3, [sp, #208] + rev w20, w9 + str w20, [sp, #172] + add w20, w9, #1 + rev w20, w20 + str w20, [sp, #188] + add w20, w9, #2 + rev w20, w20 + str w20, [sp, #204] + add w20, w9, #3 + rev w20, w20 + str w20, [sp, #220] + add w9, w9, #4 + cmp x0, x5 + b.lt Ldec_main_loop_eor3 +Ldec_prepretail_eor3: // PREPRETAIL + rev64 v4.16b, v4.16b // GHASH block 0 + rev64 v5.16b, v5.16b // GHASH block 1 + ext v11.16b, v11.16b, v11.16b, #8 // PRE 0 + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b // AES block 0 - round 0 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b // AES block 1 - round 0 + eor v4.16b, v4.16b, v11.16b // PRE 1 + rev64 v6.16b, v6.16b // GHASH block 2 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b // AES block 1 - round 1 + pmull v11.1q, v4.1d, v15.1d // GHASH block 0 - low + mov d8, v4.d[1] // GHASH block 0 - mid + pmull2 v9.1q, v4.2d, v15.2d // GHASH block 0 - high + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b // AES block 2 - round 0 + mov d10, v17.d[1] // GHASH block 0 - mid + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b // AES block 0 - round 1 + eor v8.8b, v8.8b, v4.8b // GHASH block 0 - mid + pmull2 v4.1q, v5.2d, v14.2d // GHASH block 1 - high + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b // AES block 2 - round 1 + rev64 v7.16b, v7.16b // GHASH block 3 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b // AES block 3 - round 0 + pmull v10.1q, v8.1d, v10.1d // GHASH block 0 - mid + eor v9.16b, v9.16b, v4.16b // GHASH block 1 - high + pmull v8.1q, v5.1d, v14.1d // GHASH block 1 - low + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b // AES block 3 - round 1 + mov d4, v5.d[1] // GHASH block 1 - mid + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b // AES block 0 - round 2 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b // AES block 1 - round 2 + eor v11.16b, v11.16b, v8.16b // GHASH block 1 - low + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b // AES block 2 - round 2 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b // AES block 0 - round 3 + mov d8, v6.d[1] // GHASH block 2 - mid + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b // AES block 3 - round 2 + eor v4.8b, v4.8b, v5.8b // GHASH block 1 - mid + pmull v5.1q, v6.1d, v13.1d // GHASH block 2 - low + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b // AES block 0 - round 4 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b // AES block 3 - round 3 + eor v8.8b, v8.8b, v6.8b // GHASH block 2 - mid + pmull v4.1q, v4.1d, v17.1d // GHASH block 1 - mid + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b // AES block 0 - round 5 + eor v11.16b, v11.16b, v5.16b // GHASH block 2 - low + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b // AES block 3 - round 4 + pmull2 v5.1q, v7.2d, v12.2d // GHASH block 3 - high + eor v10.16b, v10.16b, v4.16b // GHASH block 1 - mid + pmull2 v4.1q, v6.2d, v13.2d // GHASH block 2 - high + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b // AES block 3 - round 5 + ins v8.d[1], v8.d[0] // GHASH block 2 - mid + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b // AES block 2 - round 3 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b // AES block 1 - round 3 + eor3 v9.16b, v9.16b, v4.16b, v5.16b // GHASH block 2 - high & GHASH block 3 - high + pmull v4.1q, v7.1d, v12.1d // GHASH block 3 - low + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b // AES block 2 - round 4 + mov d6, v7.d[1] // GHASH block 3 - mid + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b // AES block 1 - round 4 + pmull2 v8.1q, v8.2d, v16.2d // GHASH block 2 - mid + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b // AES block 2 - round 5 + eor v6.8b, v6.8b, v7.8b // GHASH block 3 - mid + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b // AES block 1 - round 5 + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b // AES block 3 - round 6 + eor v10.16b, v10.16b, v8.16b // GHASH block 2 - mid + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b // AES block 2 - round 6 + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b // AES block 0 - round 6 + movi v8.8b, #0xc2 + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b // AES block 1 - round 6 + eor v11.16b, v11.16b, v4.16b // GHASH block 3 - low + pmull v6.1q, v6.1d, v16.1d // GHASH block 3 - mid + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b // AES block 3 - round 7 + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b // AES block 1 - round 7 + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b // AES block 0 - round 7 + eor v10.16b, v10.16b, v6.16b // GHASH block 3 - mid + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b // AES block 3 - round 8 + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b // AES block 2 - round 7 + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b // AES block 1 - round 8 + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b // AES block 0 - round 8 + shl d8, d8, #56 // mod_constant + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b // AES block 2 - round 8 + cmp w17, #12 // setup flags for AES-128/192/256 check + b.lt Ldec_finish_prepretail_eor3 // branch if AES-128 + ldp q27, q28, [x8, #144] // load rk9, rk10 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 9 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 9 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 9 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 9 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 10 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 10 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 10 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 10 + b.eq Ldec_finish_prepretail_eor3 // branch if AES-192 + ldp q27, q28, [x8, #176] // load rk11, rk12 + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b // AES block 0 - round 11 + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b // AES block 1 - round 11 + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b // AES block 2 - round 11 + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b // AES block 3 - round 11 + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b // AES block 0 - round 12 + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b // AES block 1 - round 12 + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b // AES block 2 - round 12 + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b // AES block 3 - round 12 +Ldec_finish_prepretail_eor3: + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor3 v10.16b, v10.16b, v7.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor3 v11.16b, v11.16b, v8.16b, v10.16b // MODULO - fold into low + aese v1.16b, v31.16b // AES block 1 - round N-1 + aese v0.16b, v31.16b // AES block 0 - round N-1 + aese v3.16b, v31.16b // AES block 3 - round N-1 + aese v2.16b, v31.16b // AES block 2 - round N-1 +Ldec_tail_eor3: // TAIL + sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process + ld1 { v5.16b}, [x0], #16 // AES block 0 - load ciphertext + eor v0.16b, v5.16b, v0.16b // AES block 0 - result + mov x6, v0.d[0] // AES block 0 - mov low + mov x7, v0.d[1] // AES block 0 - mov high + ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag + eor x6, x6, x13 // AES block 0 - round N low + eor x7, x7, x14 // AES block 0 - round N high + cmp x5, #48 + b.gt Ldec_blocks_more_than_3_eor3 + mov v3.16b, v2.16b + movi v10.8b, #0 + movi v11.8b, #0 + movi v9.8b, #0 + mov v2.16b, v1.16b + cmp x5, #32 + b.gt Ldec_blocks_more_than_2_eor3 + mov v3.16b, v1.16b + cmp x5, #16 + b.gt Ldec_blocks_more_than_1_eor3 + b Ldec_blocks_less_than_1_eor3 +Ldec_blocks_more_than_3_eor3: // blocks left > 3 + rev64 v4.16b, v5.16b // GHASH final-3 block + ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext + stp x6, x7, [x2], #16 // AES final-3 block - store result + mov d10, v17.d[1] // GHASH final-3 block - mid + eor v4.16b, v4.16b, v8.16b // feed in partial tag + eor v0.16b, v5.16b, v1.16b // AES final-2 block - result + mov d22, v4.d[1] // GHASH final-3 block - mid + mov x6, v0.d[0] // AES final-2 block - mov low + mov x7, v0.d[1] // AES final-2 block - mov high + eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid + movi v8.8b, #0 // suppress further partial tag feed in + pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high + pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid + eor x6, x6, x13 // AES final-2 block - round N low + pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low + eor x7, x7, x14 // AES final-2 block - round N high +Ldec_blocks_more_than_2_eor3: // blocks left > 2 + rev64 v4.16b, v5.16b // GHASH final-2 block + ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + stp x6, x7, [x2], #16 // AES final-2 block - store result + eor v0.16b, v5.16b, v2.16b // AES final-1 block - result + mov d22, v4.d[1] // GHASH final-2 block - mid + pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low + pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid + mov x6, v0.d[0] // AES final-1 block - mov low + mov x7, v0.d[1] // AES final-1 block - mov high + eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low + movi v8.8b, #0 // suppress further partial tag feed in + pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high + eor x6, x6, x13 // AES final-1 block - round N low + eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid + eor x7, x7, x14 // AES final-1 block - round N high +Ldec_blocks_more_than_1_eor3: // blocks left > 1 + stp x6, x7, [x2], #16 // AES final-1 block - store result + rev64 v4.16b, v5.16b // GHASH final-1 block + ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext + eor v4.16b, v4.16b, v8.16b // feed in partial tag + movi v8.8b, #0 // suppress further partial tag feed in + mov d22, v4.d[1] // GHASH final-1 block - mid + eor v0.16b, v5.16b, v3.16b // AES final block - result + pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high + eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid + pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low + mov x6, v0.d[0] // AES final block - mov low + ins v22.d[1], v22.d[0] // GHASH final-1 block - mid + mov x7, v0.d[1] // AES final block - mov high + pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid + eor x6, x6, x13 // AES final block - round N low + eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low + eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high + eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid + eor x7, x7, x14 // AES final block - round N high +Ldec_blocks_less_than_1_eor3: // blocks left <= 1 + add x10, x10, x1, lsr #7 // Calculate the updated counter based on the number of 16B chunks we processed + rev w10, w10 + str w10, [x16, #12] // store the updated counter + and x1, x1, #127 // bit_length %= 128 + mvn x14, xzr // rkN_h = 0xffffffffffffffff + sub x1, x1, #128 // bit_length -= 128 + mvn x13, xzr // rkN_l = 0xffffffffffffffff + ldp x4, x5, [x2] // load existing bytes we need to not overwrite + neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128]) + and x1, x1, #127 // bit_length %= 128 + lsr x14, x14, x1 // rkN_h is mask for top 64b of last block + cmp x1, #64 + csel x9, x13, x14, lt + csel x10, x14, xzr, lt + fmov d0, x9 // ctr0b is mask for last block + and x6, x6, x9 + mov v0.d[1], x10 + bic x4, x4, x9 // mask out low existing bytes + bic x5, x5, x10 // mask out high existing bytes + orr x6, x6, x4 + and x7, x7, x10 + orr x7, x7, x5 + and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits + rev64 v4.16b, v5.16b // GHASH final block + eor v4.16b, v4.16b, v8.16b // feed in partial tag + pmull v21.1q, v4.1d, v12.1d // GHASH final block - low + mov d8, v4.d[1] // GHASH final block - mid + eor v8.8b, v8.8b, v4.8b // GHASH final block - mid + pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high + pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid + eor v9.16b, v9.16b, v20.16b // GHASH final block - high + eor v11.16b, v11.16b, v21.16b // GHASH final block - low + eor v10.16b, v10.16b, v8.16b // GHASH final block - mid + ldr d8, [sp, #128] + eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up + eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up + pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid + ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment + eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid + eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid + pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low + ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment + eor v11.16b, v11.16b, v8.16b // MODULO - fold into low + stp x6, x7, [x2] + eor v11.16b, v11.16b, v10.16b // MODULO - fold into low + ext v11.16b, v11.16b, v11.16b, #8 + rev64 v11.16b, v11.16b // Final Tag + mov x0, x15 + st1 { v11.16b }, [x3] // Store final tag + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp d8, d9, [sp, #64] + ldp d10, d11, [sp, #80] + ldp d12, d13, [sp, #96] + ldp d14, d15, [sp, #112] + ldp x29, x30, [sp], #224 + AARCH64_VALIDATE_LINK_REGISTER + ret + +#endif // __ARM_MAX_ARCH__ >= 8 #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(_WIN32)
diff --git a/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc b/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc index 0f3aab2..ddd5e5a 100644 --- a/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc +++ b/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc
@@ -165,9 +165,11 @@ %define aes256gcmsiv_enc_msg_x8 BORINGSSL_PREFIX %+ _aes256gcmsiv_enc_msg_x8 %define aes256gcmsiv_kdf BORINGSSL_PREFIX %+ _aes256gcmsiv_kdf %define aes_gcm_dec_kernel BORINGSSL_PREFIX %+ _aes_gcm_dec_kernel +%define aes_gcm_dec_kernel_eor3 BORINGSSL_PREFIX %+ _aes_gcm_dec_kernel_eor3 %define aes_gcm_dec_update_vaes_avx2 BORINGSSL_PREFIX %+ _aes_gcm_dec_update_vaes_avx2 %define aes_gcm_dec_update_vaes_avx512 BORINGSSL_PREFIX %+ _aes_gcm_dec_update_vaes_avx512 %define aes_gcm_enc_kernel BORINGSSL_PREFIX %+ _aes_gcm_enc_kernel +%define aes_gcm_enc_kernel_eor3 BORINGSSL_PREFIX %+ _aes_gcm_enc_kernel_eor3 %define aes_gcm_enc_update_vaes_avx2 BORINGSSL_PREFIX %+ _aes_gcm_enc_update_vaes_avx2 %define aes_gcm_enc_update_vaes_avx512 BORINGSSL_PREFIX %+ _aes_gcm_enc_update_vaes_avx512 %define aes_hw_cbc_encrypt BORINGSSL_PREFIX %+ _aes_hw_cbc_encrypt
diff --git a/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc b/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc index 79ad399..55b655d 100644 --- a/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc +++ b/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc
@@ -165,9 +165,11 @@ %define _aes256gcmsiv_enc_msg_x8 _ %+ BORINGSSL_PREFIX %+ _aes256gcmsiv_enc_msg_x8 %define _aes256gcmsiv_kdf _ %+ BORINGSSL_PREFIX %+ _aes256gcmsiv_kdf %define _aes_gcm_dec_kernel _ %+ BORINGSSL_PREFIX %+ _aes_gcm_dec_kernel +%define _aes_gcm_dec_kernel_eor3 _ %+ BORINGSSL_PREFIX %+ _aes_gcm_dec_kernel_eor3 %define _aes_gcm_dec_update_vaes_avx2 _ %+ BORINGSSL_PREFIX %+ _aes_gcm_dec_update_vaes_avx2 %define _aes_gcm_dec_update_vaes_avx512 _ %+ BORINGSSL_PREFIX %+ _aes_gcm_dec_update_vaes_avx512 %define _aes_gcm_enc_kernel _ %+ BORINGSSL_PREFIX %+ _aes_gcm_enc_kernel +%define _aes_gcm_enc_kernel_eor3 _ %+ BORINGSSL_PREFIX %+ _aes_gcm_enc_kernel_eor3 %define _aes_gcm_enc_update_vaes_avx2 _ %+ BORINGSSL_PREFIX %+ _aes_gcm_enc_update_vaes_avx2 %define _aes_gcm_enc_update_vaes_avx512 _ %+ BORINGSSL_PREFIX %+ _aes_gcm_enc_update_vaes_avx512 %define _aes_hw_cbc_encrypt _ %+ BORINGSSL_PREFIX %+ _aes_hw_cbc_encrypt
diff --git a/gen/sources.bzl b/gen/sources.bzl index 68b393c..c558749 100644 --- a/gen/sources.bzl +++ b/gen/sources.bzl
@@ -625,6 +625,7 @@ ] crypto_internal_headers = [ + "crypto/armv8_feature_parsing.h", "crypto/asn1/internal.h", "crypto/bcm_support.h", "crypto/bio/internal.h",
diff --git a/gen/sources.cmake b/gen/sources.cmake index 76e4935..28b63d7 100644 --- a/gen/sources.cmake +++ b/gen/sources.cmake
@@ -647,6 +647,7 @@ set( CRYPTO_INTERNAL_HEADERS + crypto/armv8_feature_parsing.h crypto/asn1/internal.h crypto/bcm_support.h crypto/bio/internal.h
diff --git a/gen/sources.gni b/gen/sources.gni index 1b7aa00..eb83060 100644 --- a/gen/sources.gni +++ b/gen/sources.gni
@@ -625,6 +625,7 @@ ] crypto_internal_headers = [ + "crypto/armv8_feature_parsing.h", "crypto/asn1/internal.h", "crypto/bcm_support.h", "crypto/bio/internal.h",
diff --git a/gen/sources.json b/gen/sources.json index ce8185a..6c08a9c 100644 --- a/gen/sources.json +++ b/gen/sources.json
@@ -607,6 +607,7 @@ "include/openssl/xwing.h" ], "internal_hdrs": [ + "crypto/armv8_feature_parsing.h", "crypto/asn1/internal.h", "crypto/bcm_support.h", "crypto/bio/internal.h",
diff --git a/gen/sources.mk b/gen/sources.mk index c7b695a..d0b18b2 100644 --- a/gen/sources.mk +++ b/gen/sources.mk
@@ -615,6 +615,7 @@ include/openssl/xwing.h boringssl_crypto_internal_headers := \ + crypto/armv8_feature_parsing.h \ crypto/asn1/internal.h \ crypto/bcm_support.h \ crypto/bio/internal.h \
diff --git a/include/openssl/prefix_symbols_internal_S.h b/include/openssl/prefix_symbols_internal_S.h index f0d8142..31d21ac 100644 --- a/include/openssl/prefix_symbols_internal_S.h +++ b/include/openssl/prefix_symbols_internal_S.h
@@ -169,9 +169,11 @@ #define _aes256gcmsiv_enc_msg_x8 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes256gcmsiv_enc_msg_x8)) #define _aes256gcmsiv_kdf BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes256gcmsiv_kdf)) #define _aes_gcm_dec_kernel BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_dec_kernel)) +#define _aes_gcm_dec_kernel_eor3 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_dec_kernel_eor3)) #define _aes_gcm_dec_update_vaes_avx2 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_dec_update_vaes_avx2)) #define _aes_gcm_dec_update_vaes_avx512 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_dec_update_vaes_avx512)) #define _aes_gcm_enc_kernel BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_enc_kernel)) +#define _aes_gcm_enc_kernel_eor3 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_enc_kernel_eor3)) #define _aes_gcm_enc_update_vaes_avx2 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_enc_update_vaes_avx2)) #define _aes_gcm_enc_update_vaes_avx512 BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_gcm_enc_update_vaes_avx512)) #define _aes_hw_cbc_encrypt BORINGSSL_SYMBOL(BORINGSSL_ADD_PREFIX(aes_hw_cbc_encrypt)) @@ -469,9 +471,11 @@ #define aes256gcmsiv_enc_msg_x8 BORINGSSL_ADD_PREFIX(aes256gcmsiv_enc_msg_x8) #define aes256gcmsiv_kdf BORINGSSL_ADD_PREFIX(aes256gcmsiv_kdf) #define aes_gcm_dec_kernel BORINGSSL_ADD_PREFIX(aes_gcm_dec_kernel) +#define aes_gcm_dec_kernel_eor3 BORINGSSL_ADD_PREFIX(aes_gcm_dec_kernel_eor3) #define aes_gcm_dec_update_vaes_avx2 BORINGSSL_ADD_PREFIX(aes_gcm_dec_update_vaes_avx2) #define aes_gcm_dec_update_vaes_avx512 BORINGSSL_ADD_PREFIX(aes_gcm_dec_update_vaes_avx512) #define aes_gcm_enc_kernel BORINGSSL_ADD_PREFIX(aes_gcm_enc_kernel) +#define aes_gcm_enc_kernel_eor3 BORINGSSL_ADD_PREFIX(aes_gcm_enc_kernel_eor3) #define aes_gcm_enc_update_vaes_avx2 BORINGSSL_ADD_PREFIX(aes_gcm_enc_update_vaes_avx2) #define aes_gcm_enc_update_vaes_avx512 BORINGSSL_ADD_PREFIX(aes_gcm_enc_update_vaes_avx512) #define aes_hw_cbc_encrypt BORINGSSL_ADD_PREFIX(aes_hw_cbc_encrypt)
diff --git a/include/openssl/prefix_symbols_internal_c.h b/include/openssl/prefix_symbols_internal_c.h index 2332027..04ad11d 100644 --- a/include/openssl/prefix_symbols_internal_c.h +++ b/include/openssl/prefix_symbols_internal_c.h
@@ -167,9 +167,11 @@ #define aes256gcmsiv_enc_msg_x8 BORINGSSL_ADD_PREFIX(aes256gcmsiv_enc_msg_x8) #define aes256gcmsiv_kdf BORINGSSL_ADD_PREFIX(aes256gcmsiv_kdf) #define aes_gcm_dec_kernel BORINGSSL_ADD_PREFIX(aes_gcm_dec_kernel) +#define aes_gcm_dec_kernel_eor3 BORINGSSL_ADD_PREFIX(aes_gcm_dec_kernel_eor3) #define aes_gcm_dec_update_vaes_avx2 BORINGSSL_ADD_PREFIX(aes_gcm_dec_update_vaes_avx2) #define aes_gcm_dec_update_vaes_avx512 BORINGSSL_ADD_PREFIX(aes_gcm_dec_update_vaes_avx512) #define aes_gcm_enc_kernel BORINGSSL_ADD_PREFIX(aes_gcm_enc_kernel) +#define aes_gcm_enc_kernel_eor3 BORINGSSL_ADD_PREFIX(aes_gcm_enc_kernel_eor3) #define aes_gcm_enc_update_vaes_avx2 BORINGSSL_ADD_PREFIX(aes_gcm_enc_update_vaes_avx2) #define aes_gcm_enc_update_vaes_avx512 BORINGSSL_ADD_PREFIX(aes_gcm_enc_update_vaes_avx512) #define aes_hw_cbc_encrypt BORINGSSL_ADD_PREFIX(aes_hw_cbc_encrypt)