Collapse the modes directory into aes Before further surgery on aes for BCM purposes. Bug: 392625968 Change-Id: If87ef7c391ef46b09d2b68ddd1065810a71f0bf9 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/75787 Auto-Submit: Bob Beck <bbe@google.com> Reviewed-by: David Benjamin <davidben@google.com> Commit-Queue: Bob Beck <bbe@google.com>
diff --git a/build.json b/build.json index 8daf6a9..a2693eb 100644 --- a/build.json +++ b/build.json
@@ -18,8 +18,15 @@ "internal_hdrs": [ "crypto/fipsmodule/aes/aes.cc.inc", "crypto/fipsmodule/aes/aes_nohw.cc.inc", + "crypto/fipsmodule/aes/cbc.cc.inc", + "crypto/fipsmodule/aes/cfb.cc.inc", + "crypto/fipsmodule/aes/ctr.cc.inc", + "crypto/fipsmodule/aes/gcm.cc.inc", + "crypto/fipsmodule/aes/gcm_nohw.cc.inc", "crypto/fipsmodule/aes/key_wrap.cc.inc", "crypto/fipsmodule/aes/mode_wrappers.cc.inc", + "crypto/fipsmodule/aes/ofb.cc.inc", + "crypto/fipsmodule/aes/polyval.cc.inc", "crypto/fipsmodule/bn/add.cc.inc", "crypto/fipsmodule/bn/asm/x86_64-gcc.cc.inc", "crypto/fipsmodule/bn/bn.cc.inc", @@ -71,13 +78,6 @@ "crypto/fipsmodule/keccak/keccak.cc.inc", "crypto/fipsmodule/mldsa/mldsa.cc.inc", "crypto/fipsmodule/mlkem/mlkem.cc.inc", - "crypto/fipsmodule/modes/cbc.cc.inc", - "crypto/fipsmodule/modes/cfb.cc.inc", - "crypto/fipsmodule/modes/ctr.cc.inc", - "crypto/fipsmodule/modes/gcm.cc.inc", - "crypto/fipsmodule/modes/gcm_nohw.cc.inc", - "crypto/fipsmodule/modes/ofb.cc.inc", - "crypto/fipsmodule/modes/polyval.cc.inc", "crypto/fipsmodule/rand/ctrdrbg.cc.inc", "crypto/fipsmodule/rand/rand.cc.inc", "crypto/fipsmodule/rsa/blinding.cc.inc", @@ -103,11 +103,11 @@ ], "perlasm_aarch64": [ {"src": "crypto/fipsmodule/aes/asm/aesv8-armx.pl", "dst": "aesv8-armv8"}, - {"src": "crypto/fipsmodule/modes/asm/aesv8-gcm-armv8.pl"}, + {"src": "crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl"}, {"src": "crypto/fipsmodule/bn/asm/armv8-mont.pl"}, {"src": "crypto/fipsmodule/bn/asm/bn-armv8.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghash-neon-armv8.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghashv8-armx.pl", "dst": "ghashv8-armv8"}, + {"src": "crypto/fipsmodule/aes/asm/ghash-neon-armv8.pl"}, + {"src": "crypto/fipsmodule/aes/asm/ghashv8-armx.pl", "dst": "ghashv8-armv8"}, {"src": "crypto/fipsmodule/ec/asm/p256_beeu-armv8-asm.pl"}, {"src": "crypto/fipsmodule/ec/asm/p256-armv8-asm.pl"}, {"src": "crypto/fipsmodule/sha/asm/sha1-armv8.pl"}, @@ -119,8 +119,8 @@ {"src": "crypto/fipsmodule/aes/asm/aesv8-armx.pl", "dst": "aesv8-armv7"}, {"src": "crypto/fipsmodule/bn/asm/armv4-mont.pl"}, {"src": "crypto/fipsmodule/aes/asm/bsaes-armv7.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghash-armv4.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghashv8-armx.pl", "dst": "ghashv8-armv7"}, + {"src": "crypto/fipsmodule/aes/asm/ghash-armv4.pl"}, + {"src": "crypto/fipsmodule/aes/asm/ghashv8-armx.pl", "dst": "ghashv8-armv7"}, {"src": "crypto/fipsmodule/sha/asm/sha1-armv4-large.pl"}, {"src": "crypto/fipsmodule/sha/asm/sha256-armv4.pl"}, {"src": "crypto/fipsmodule/sha/asm/sha512-armv4.pl"}, @@ -130,8 +130,8 @@ {"src": "crypto/fipsmodule/aes/asm/aesni-x86.pl"}, {"src": "crypto/fipsmodule/bn/asm/bn-586.pl"}, {"src": "crypto/fipsmodule/bn/asm/co-586.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghash-ssse3-x86.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghash-x86.pl"}, + {"src": "crypto/fipsmodule/aes/asm/ghash-ssse3-x86.pl"}, + {"src": "crypto/fipsmodule/aes/asm/ghash-x86.pl"}, {"src": "crypto/fipsmodule/sha/asm/sha1-586.pl"}, {"src": "crypto/fipsmodule/sha/asm/sha256-586.pl"}, {"src": "crypto/fipsmodule/sha/asm/sha512-586.pl"}, @@ -139,12 +139,12 @@ {"src": "crypto/fipsmodule/bn/asm/x86-mont.pl"} ], "perlasm_x86_64": [ - {"src": "crypto/fipsmodule/modes/asm/aesni-gcm-x86_64.pl"}, - {"src": "crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl"}, - {"src": "crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl"}, + {"src": "crypto/fipsmodule/aes/asm/aesni-gcm-x86_64.pl"}, + {"src": "crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl"}, + {"src": "crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl"}, {"src": "crypto/fipsmodule/aes/asm/aesni-x86_64.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghash-ssse3-x86_64.pl"}, - {"src": "crypto/fipsmodule/modes/asm/ghash-x86_64.pl"}, + {"src": "crypto/fipsmodule/aes/asm/ghash-ssse3-x86_64.pl"}, + {"src": "crypto/fipsmodule/aes/asm/ghash-x86_64.pl"}, {"src": "crypto/fipsmodule/ec/asm/p256_beeu-x86_64-asm.pl"}, {"src": "crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl"}, {"src": "crypto/fipsmodule/rand/asm/rdrand-x86_64.pl"}, @@ -516,7 +516,6 @@ "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/keccak/internal.h", - "crypto/fipsmodule/modes/internal.h", "crypto/fipsmodule/rand/internal.h", "crypto/fipsmodule/rsa/internal.h", "crypto/fipsmodule/service_indicator/internal.h", @@ -825,6 +824,7 @@ "crypto/evp/pbkdf_test.cc", "crypto/evp/scrypt_test.cc", "crypto/fipsmodule/aes/aes_test.cc", + "crypto/fipsmodule/aes/gcm_test.cc", "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", @@ -833,7 +833,6 @@ "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/hkdf/hkdf_test.cc", "crypto/fipsmodule/keccak/keccak_test.cc", - "crypto/fipsmodule/modes/gcm_test.cc", "crypto/fipsmodule/rand/ctrdrbg_test.cc", "crypto/fipsmodule/service_indicator/service_indicator_test.cc", "crypto/fipsmodule/sha/sha_test.cc",
diff --git a/crypto/fipsmodule/aes/aes.cc.inc b/crypto/fipsmodule/aes/aes.cc.inc index bcdc4dc..f1737cd 100644 --- a/crypto/fipsmodule/aes/aes.cc.inc +++ b/crypto/fipsmodule/aes/aes.cc.inc
@@ -12,7 +12,6 @@ #include <assert.h> #include "internal.h" -#include "../modes/internal.h" // Be aware that different sets of AES functions use incompatible key
diff --git a/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl b/crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl rename to crypto/fipsmodule/aes/asm/aes-gcm-avx10-x86_64.pl
diff --git a/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl b/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl rename to crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl
diff --git a/crypto/fipsmodule/modes/asm/aesni-gcm-x86_64.pl b/crypto/fipsmodule/aes/asm/aesni-gcm-x86_64.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/aesni-gcm-x86_64.pl rename to crypto/fipsmodule/aes/asm/aesni-gcm-x86_64.pl
diff --git a/crypto/fipsmodule/modes/asm/aesv8-gcm-armv8.pl b/crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/aesv8-gcm-armv8.pl rename to crypto/fipsmodule/aes/asm/aesv8-gcm-armv8.pl
diff --git a/crypto/fipsmodule/modes/asm/ghash-armv4.pl b/crypto/fipsmodule/aes/asm/ghash-armv4.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghash-armv4.pl rename to crypto/fipsmodule/aes/asm/ghash-armv4.pl
diff --git a/crypto/fipsmodule/modes/asm/ghash-neon-armv8.pl b/crypto/fipsmodule/aes/asm/ghash-neon-armv8.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghash-neon-armv8.pl rename to crypto/fipsmodule/aes/asm/ghash-neon-armv8.pl
diff --git a/crypto/fipsmodule/modes/asm/ghash-ssse3-x86.pl b/crypto/fipsmodule/aes/asm/ghash-ssse3-x86.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghash-ssse3-x86.pl rename to crypto/fipsmodule/aes/asm/ghash-ssse3-x86.pl
diff --git a/crypto/fipsmodule/modes/asm/ghash-ssse3-x86_64.pl b/crypto/fipsmodule/aes/asm/ghash-ssse3-x86_64.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghash-ssse3-x86_64.pl rename to crypto/fipsmodule/aes/asm/ghash-ssse3-x86_64.pl
diff --git a/crypto/fipsmodule/modes/asm/ghash-x86.pl b/crypto/fipsmodule/aes/asm/ghash-x86.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghash-x86.pl rename to crypto/fipsmodule/aes/asm/ghash-x86.pl
diff --git a/crypto/fipsmodule/modes/asm/ghash-x86_64.pl b/crypto/fipsmodule/aes/asm/ghash-x86_64.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghash-x86_64.pl rename to crypto/fipsmodule/aes/asm/ghash-x86_64.pl
diff --git a/crypto/fipsmodule/modes/asm/ghashv8-armx.pl b/crypto/fipsmodule/aes/asm/ghashv8-armx.pl similarity index 100% rename from crypto/fipsmodule/modes/asm/ghashv8-armx.pl rename to crypto/fipsmodule/aes/asm/ghashv8-armx.pl
diff --git a/crypto/fipsmodule/modes/cbc.cc.inc b/crypto/fipsmodule/aes/cbc.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/cbc.cc.inc rename to crypto/fipsmodule/aes/cbc.cc.inc
diff --git a/crypto/fipsmodule/modes/cfb.cc.inc b/crypto/fipsmodule/aes/cfb.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/cfb.cc.inc rename to crypto/fipsmodule/aes/cfb.cc.inc
diff --git a/crypto/fipsmodule/modes/ctr.cc.inc b/crypto/fipsmodule/aes/ctr.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/ctr.cc.inc rename to crypto/fipsmodule/aes/ctr.cc.inc
diff --git a/crypto/fipsmodule/modes/gcm.cc.inc b/crypto/fipsmodule/aes/gcm.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/gcm.cc.inc rename to crypto/fipsmodule/aes/gcm.cc.inc
diff --git a/crypto/fipsmodule/modes/gcm_nohw.cc.inc b/crypto/fipsmodule/aes/gcm_nohw.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/gcm_nohw.cc.inc rename to crypto/fipsmodule/aes/gcm_nohw.cc.inc
diff --git a/crypto/fipsmodule/modes/gcm_test.cc b/crypto/fipsmodule/aes/gcm_test.cc similarity index 100% rename from crypto/fipsmodule/modes/gcm_test.cc rename to crypto/fipsmodule/aes/gcm_test.cc
diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index 6d56b11..29b7d05 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h
@@ -268,6 +268,335 @@ void aes_nohw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, const AES_KEY *key, uint8_t *ivec, int enc); +// Modes + +inline void CRYPTO_xor16(uint8_t out[16], const uint8_t a[16], + const uint8_t b[16]) { + // TODO(davidben): Ideally we'd leave this to the compiler, which could use + // vector registers, etc. But the compiler doesn't know that |in| and |out| + // cannot partially alias. |restrict| is slightly two strict (we allow exact + // aliasing), but perhaps in-place could be a separate function? + static_assert(16 % sizeof(crypto_word_t) == 0, + "block cannot be evenly divided into words"); + for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) { + CRYPTO_store_word_le( + out + i, CRYPTO_load_word_le(a + i) ^ CRYPTO_load_word_le(b + i)); + } +} + + +// CTR. + +// CRYPTO_ctr128_encrypt_ctr32 encrypts (or decrypts, it's the same in CTR mode) +// |len| bytes from |in| to |out| using |block| in counter mode. There's no +// requirement that |len| be a multiple of any value and any partial blocks are +// stored in |ecount_buf| and |*num|, which must be zeroed before the initial +// call. The counter is a 128-bit, big-endian value in |ivec| and is +// incremented by this function. If the counter overflows, it wraps around. +// |ctr| must be a function that performs CTR mode but only deals with the lower +// 32 bits of the counter. +void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + uint8_t ecount_buf[16], unsigned *num, + ctr128_f ctr); + + +// GCM. +// +// This API differs from the upstream API slightly. The |GCM128_CONTEXT| does +// not have a |key| pointer that points to the key as upstream's version does. +// Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT| +// can be safely copied. Additionally, |gcm_key| is split into a separate +// struct. + +// gcm_impl_t specifies an assembly implementation of AES-GCM. +enum gcm_impl_t { + gcm_separate = 0, // No combined AES-GCM, but may have AES-CTR and GHASH. + gcm_x86_aesni, + gcm_x86_vaes_avx2, + gcm_x86_vaes_avx10_512, + gcm_arm64_aes, +}; + +typedef struct { uint64_t hi,lo; } u128; + +// gmult_func multiplies |Xi| by the GCM key and writes the result back to +// |Xi|. +typedef void (*gmult_func)(uint8_t Xi[16], const u128 Htable[16]); + +// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from +// |inp|. The result is written back to |Xi| and the |len| argument must be a +// multiple of 16. +typedef void (*ghash_func)(uint8_t Xi[16], const u128 Htable[16], + const uint8_t *inp, size_t len); + +typedef struct gcm128_key_st { + u128 Htable[16]; + gmult_func gmult; + ghash_func ghash; + AES_KEY aes; + + ctr128_f ctr; + block128_f block; + enum gcm_impl_t impl; +} GCM128_KEY; + +// GCM128_CONTEXT contains state for a single GCM operation. The structure +// should be zero-initialized before use. +typedef struct { + // The following 5 names follow names in GCM specification + uint8_t Yi[16]; + uint8_t EKi[16]; + uint8_t EK0[16]; + struct { + uint64_t aad; + uint64_t msg; + } len; + uint8_t Xi[16]; + unsigned mres, ares; +} GCM128_CONTEXT; + +#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) +// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is +// used. +int crypto_gcm_clmul_enabled(void); +#endif + +// CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to +// |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware +// accelerated) functions for performing operations in the GHASH field. +void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash, + u128 out_table[16], const uint8_t gcm_key[16]); + +// CRYPTO_gcm128_init_aes_key initialises |gcm_key| to with AES key |key|. +void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key, + size_t key_bytes); + +// CRYPTO_gcm128_init_ctx initializes |ctx| to encrypt with |key| and |iv|. +void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx, + const uint8_t *iv, size_t iv_len); + +// CRYPTO_gcm128_aad adds to the authenticated data for an instance of GCM. +// This must be called before and data is encrypted. |key| must be the same +// value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on success +// and zero otherwise. +int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx, + const uint8_t *aad, size_t aad_len); + +// CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. |key| must be +// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on +// success and zero otherwise. +int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx, + const uint8_t *in, uint8_t *out, size_t len); + +// CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. |key| must be +// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on +// success and zero otherwise. +int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx, + const uint8_t *in, uint8_t *out, size_t len); + +// CRYPTO_gcm128_finish calculates the authenticator and compares it against +// |len| bytes of |tag|. |key| must be the same value that was passed to +// |CRYPTO_gcm128_init_ctx|. It returns one on success and zero otherwise. +int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx, + const uint8_t *tag, size_t len); + +// CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|. +// The minimum of |len| and 16 bytes are copied into |tag|. |key| must be the +// same value that was passed to |CRYPTO_gcm128_init_ctx|. +void CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx, uint8_t *tag, + size_t len); + + +// GCM assembly. + +void gcm_init_nohw(u128 Htable[16], const uint64_t H[2]); +void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, + size_t len); + +#if !defined(OPENSSL_NO_ASM) + +#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) +#define GCM_FUNCREF +void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]); +void gcm_gmult_clmul(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_clmul(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, + size_t len); + +void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]); +void gcm_gmult_ssse3(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_ssse3(uint8_t Xi[16], const u128 Htable[16], const uint8_t *in, + size_t len); + +#if defined(OPENSSL_X86_64) +#define GHASH_ASM_X86_64 +void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]); +void gcm_gmult_avx(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_avx(uint8_t Xi[16], const u128 Htable[16], const uint8_t *in, + size_t len); + +#define HW_GCM +size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + const u128 Htable[16], uint8_t Xi[16]); +size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + const u128 Htable[16], uint8_t Xi[16]); + +void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]); +void gcm_gmult_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16], + const uint8_t *in, size_t len); +void aes_gcm_enc_update_vaes_avx2(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, const uint8_t ivec[16], + const u128 Htable[16], uint8_t Xi[16]); +void aes_gcm_dec_update_vaes_avx2(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, const uint8_t ivec[16], + const u128 Htable[16], uint8_t Xi[16]); + +void gcm_init_vpclmulqdq_avx10_512(u128 Htable[16], const uint64_t H[2]); +void gcm_gmult_vpclmulqdq_avx10(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_vpclmulqdq_avx10_512(uint8_t Xi[16], const u128 Htable[16], + const uint8_t *in, size_t len); +void aes_gcm_enc_update_vaes_avx10_512(const uint8_t *in, uint8_t *out, + size_t len, const AES_KEY *key, + const uint8_t ivec[16], + const u128 Htable[16], uint8_t Xi[16]); +void aes_gcm_dec_update_vaes_avx10_512(const uint8_t *in, uint8_t *out, + size_t len, const AES_KEY *key, + const uint8_t ivec[16], + const u128 Htable[16], uint8_t Xi[16]); + +#endif // OPENSSL_X86_64 + +#if defined(OPENSSL_X86) +#define GHASH_ASM_X86 +#endif // OPENSSL_X86 + +#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) + +#define GHASH_ASM_ARM +#define GCM_FUNCREF + +inline int gcm_pmull_capable(void) { return CRYPTO_is_ARMv8_PMULL_capable(); } + +void gcm_init_v8(u128 Htable[16], const uint64_t H[2]); +void gcm_gmult_v8(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_v8(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, + size_t len); + +inline int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); } + +void gcm_init_neon(u128 Htable[16], const uint64_t H[2]); +void gcm_gmult_neon(uint8_t Xi[16], const u128 Htable[16]); +void gcm_ghash_neon(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, + size_t len); + +#if defined(OPENSSL_AARCH64) +#define HW_GCM +// These functions are defined in aesv8-gcm-armv8.pl. +void aes_gcm_enc_kernel(const uint8_t *in, uint64_t in_bits, void *out, + void *Xi, uint8_t *ivec, const AES_KEY *key, + const u128 Htable[16]); +void aes_gcm_dec_kernel(const uint8_t *in, uint64_t in_bits, void *out, + void *Xi, uint8_t *ivec, const AES_KEY *key, + const u128 Htable[16]); +#endif + +#endif +#endif // OPENSSL_NO_ASM + + +// CBC. + +// cbc128_f is the type of a function that performs CBC-mode encryption. +typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], int enc); + +// CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the +// given IV and block cipher in CBC mode. The input need not be a multiple of +// 128 bits long, but the output will round up to the nearest 128 bit multiple, +// zero padding the input if needed. The IV will be updated on return. +void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + block128_f block); + +// CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the +// given IV and block cipher in CBC mode. If |len| is not a multiple of 128 +// bits then only that many bytes will be written, but a multiple of 128 bits +// is always read from |in|. The IV will be updated on return. +void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + block128_f block); + + +// OFB. + +// CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode) +// |len| bytes from |in| to |out| using |block| in OFB mode. There's no +// requirement that |len| be a multiple of any value and any partial blocks are +// stored in |ivec| and |*num|, the latter must be zero before the initial +// call. +void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], unsigned *num, + block128_f block); + + +// CFB. + +// CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes +// from |in| to |out| using |block| in CFB mode. There's no requirement that +// |len| be a multiple of any value and any partial blocks are stored in |ivec| +// and |*num|, the latter must be zero before the initial call. +void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], unsigned *num, + int enc, block128_f block); + +// CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes +// from |in| to |out| using |block| in CFB-8 mode. Prior to the first call +// |num| should be set to zero. +void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + unsigned *num, int enc, block128_f block); + +// CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes +// from |in| to |out| using |block| in CFB-1 mode. Prior to the first call +// |num| should be set to zero. +void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits, + const AES_KEY *key, uint8_t ivec[16], + unsigned *num, int enc, block128_f block); + +size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t ivec[16], + block128_f block); + + +// POLYVAL. +// +// POLYVAL is a polynomial authenticator that operates over a field very +// similar to the one that GHASH uses. See +// https://www.rfc-editor.org/rfc/rfc8452.html#section-3. + +struct polyval_ctx { + uint8_t S[16]; + u128 Htable[16]; + gmult_func gmult; + ghash_func ghash; +}; + +// CRYPTO_POLYVAL_init initialises |ctx| using |key|. +void CRYPTO_POLYVAL_init(struct polyval_ctx *ctx, const uint8_t key[16]); + +// CRYPTO_POLYVAL_update_blocks updates the accumulator in |ctx| given the +// blocks from |in|. Only a whole number of blocks can be processed so |in_len| +// must be a multiple of 16. +void CRYPTO_POLYVAL_update_blocks(struct polyval_ctx *ctx, const uint8_t *in, + size_t in_len); + +// CRYPTO_POLYVAL_finish writes the accumulator from |ctx| to |out|. +void CRYPTO_POLYVAL_finish(const struct polyval_ctx *ctx, uint8_t out[16]); + } // extern C
diff --git a/crypto/fipsmodule/aes/mode_wrappers.cc.inc b/crypto/fipsmodule/aes/mode_wrappers.cc.inc index cc1fd8a..4c5b5e1 100644 --- a/crypto/fipsmodule/aes/mode_wrappers.cc.inc +++ b/crypto/fipsmodule/aes/mode_wrappers.cc.inc
@@ -12,7 +12,6 @@ #include <assert.h> #include "../aes/internal.h" -#include "../modes/internal.h" #include "../service_indicator/internal.h"
diff --git a/crypto/fipsmodule/modes/ofb.cc.inc b/crypto/fipsmodule/aes/ofb.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/ofb.cc.inc rename to crypto/fipsmodule/aes/ofb.cc.inc
diff --git a/crypto/fipsmodule/modes/polyval.cc.inc b/crypto/fipsmodule/aes/polyval.cc.inc similarity index 100% rename from crypto/fipsmodule/modes/polyval.cc.inc rename to crypto/fipsmodule/aes/polyval.cc.inc
diff --git a/crypto/fipsmodule/bcm.cc b/crypto/fipsmodule/bcm.cc index d791da5..2c2b5c9 100644 --- a/crypto/fipsmodule/bcm.cc +++ b/crypto/fipsmodule/bcm.cc
@@ -36,8 +36,15 @@ // separate compilation units again. #include "aes/aes.cc.inc" #include "aes/aes_nohw.cc.inc" +#include "aes/cbc.cc.inc" +#include "aes/cfb.cc.inc" +#include "aes/ctr.cc.inc" +#include "aes/gcm.cc.inc" +#include "aes/gcm_nohw.cc.inc" #include "aes/key_wrap.cc.inc" #include "aes/mode_wrappers.cc.inc" +#include "aes/ofb.cc.inc" +#include "aes/polyval.cc.inc" #include "bn/add.cc.inc" #include "bn/asm/x86_64-gcc.cc.inc" #include "bn/bn.cc.inc" @@ -89,13 +96,6 @@ #include "keccak/keccak.cc.inc" #include "mldsa/mldsa.cc.inc" #include "mlkem/mlkem.cc.inc" -#include "modes/cbc.cc.inc" -#include "modes/cfb.cc.inc" -#include "modes/ctr.cc.inc" -#include "modes/gcm.cc.inc" -#include "modes/gcm_nohw.cc.inc" -#include "modes/ofb.cc.inc" -#include "modes/polyval.cc.inc" #include "rand/ctrdrbg.cc.inc" #include "rand/rand.cc.inc" #include "rsa/blinding.cc.inc"
diff --git a/crypto/fipsmodule/cipher/e_aes.cc.inc b/crypto/fipsmodule/cipher/e_aes.cc.inc index 1f26418..cd46e9e 100644 --- a/crypto/fipsmodule/cipher/e_aes.cc.inc +++ b/crypto/fipsmodule/cipher/e_aes.cc.inc
@@ -22,7 +22,6 @@ #include "../aes/internal.h" #include "../bcm_interface.h" #include "../delocate.h" -#include "../modes/internal.h" #include "../service_indicator/internal.h" #include "internal.h"
diff --git a/crypto/fipsmodule/cipher/e_aesccm.cc.inc b/crypto/fipsmodule/cipher/e_aesccm.cc.inc index a5d324e..04820a2 100644 --- a/crypto/fipsmodule/cipher/e_aesccm.cc.inc +++ b/crypto/fipsmodule/cipher/e_aesccm.cc.inc
@@ -17,7 +17,6 @@ #include "../aes/internal.h" #include "../delocate.h" -#include "../modes/internal.h" #include "../service_indicator/internal.h" #include "internal.h"
diff --git a/crypto/fipsmodule/cipher/internal.h b/crypto/fipsmodule/cipher/internal.h index 9aa0ac5..cb8c999 100644 --- a/crypto/fipsmodule/cipher/internal.h +++ b/crypto/fipsmodule/cipher/internal.h
@@ -16,7 +16,7 @@ #include <openssl/aes.h> #include "../../internal.h" -#include "../modes/internal.h" +#include "../aes/internal.h" #if defined(__cplusplus) extern "C" {
diff --git a/crypto/fipsmodule/modes/internal.h b/crypto/fipsmodule/modes/internal.h deleted file mode 100644 index 7a6e9aa..0000000 --- a/crypto/fipsmodule/modes/internal.h +++ /dev/null
@@ -1,361 +0,0 @@ -/* - * Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_HEADER_MODES_INTERNAL_H -#define OPENSSL_HEADER_MODES_INTERNAL_H - -#include <openssl/base.h> - -#include <openssl/aes.h> - -#include <assert.h> -#include <stdlib.h> -#include <string.h> - -#include "../../internal.h" -#include "../aes/internal.h" - -#if defined(__cplusplus) -extern "C" { -#endif - - -inline void CRYPTO_xor16(uint8_t out[16], const uint8_t a[16], - const uint8_t b[16]) { - // TODO(davidben): Ideally we'd leave this to the compiler, which could use - // vector registers, etc. But the compiler doesn't know that |in| and |out| - // cannot partially alias. |restrict| is slightly two strict (we allow exact - // aliasing), but perhaps in-place could be a separate function? - static_assert(16 % sizeof(crypto_word_t) == 0, - "block cannot be evenly divided into words"); - for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) { - CRYPTO_store_word_le( - out + i, CRYPTO_load_word_le(a + i) ^ CRYPTO_load_word_le(b + i)); - } -} - - -// CTR. - -// CRYPTO_ctr128_encrypt_ctr32 encrypts (or decrypts, it's the same in CTR mode) -// |len| bytes from |in| to |out| using |block| in counter mode. There's no -// requirement that |len| be a multiple of any value and any partial blocks are -// stored in |ecount_buf| and |*num|, which must be zeroed before the initial -// call. The counter is a 128-bit, big-endian value in |ivec| and is -// incremented by this function. If the counter overflows, it wraps around. -// |ctr| must be a function that performs CTR mode but only deals with the lower -// 32 bits of the counter. -void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - uint8_t ecount_buf[16], unsigned *num, - ctr128_f ctr); - - -// GCM. -// -// This API differs from the upstream API slightly. The |GCM128_CONTEXT| does -// not have a |key| pointer that points to the key as upstream's version does. -// Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT| -// can be safely copied. Additionally, |gcm_key| is split into a separate -// struct. - -// gcm_impl_t specifies an assembly implementation of AES-GCM. -enum gcm_impl_t { - gcm_separate = 0, // No combined AES-GCM, but may have AES-CTR and GHASH. - gcm_x86_aesni, - gcm_x86_vaes_avx2, - gcm_x86_vaes_avx10_512, - gcm_arm64_aes, -}; - -typedef struct { uint64_t hi,lo; } u128; - -// gmult_func multiplies |Xi| by the GCM key and writes the result back to -// |Xi|. -typedef void (*gmult_func)(uint8_t Xi[16], const u128 Htable[16]); - -// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from -// |inp|. The result is written back to |Xi| and the |len| argument must be a -// multiple of 16. -typedef void (*ghash_func)(uint8_t Xi[16], const u128 Htable[16], - const uint8_t *inp, size_t len); - -typedef struct gcm128_key_st { - u128 Htable[16]; - gmult_func gmult; - ghash_func ghash; - AES_KEY aes; - - ctr128_f ctr; - block128_f block; - enum gcm_impl_t impl; -} GCM128_KEY; - -// GCM128_CONTEXT contains state for a single GCM operation. The structure -// should be zero-initialized before use. -typedef struct { - // The following 5 names follow names in GCM specification - uint8_t Yi[16]; - uint8_t EKi[16]; - uint8_t EK0[16]; - struct { - uint64_t aad; - uint64_t msg; - } len; - uint8_t Xi[16]; - unsigned mres, ares; -} GCM128_CONTEXT; - -#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) -// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is -// used. -int crypto_gcm_clmul_enabled(void); -#endif - -// CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to -// |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware -// accelerated) functions for performing operations in the GHASH field. -void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash, - u128 out_table[16], const uint8_t gcm_key[16]); - -// CRYPTO_gcm128_init_aes_key initialises |gcm_key| to with AES key |key|. -void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key, - size_t key_bytes); - -// CRYPTO_gcm128_init_ctx initializes |ctx| to encrypt with |key| and |iv|. -void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx, - const uint8_t *iv, size_t iv_len); - -// CRYPTO_gcm128_aad adds to the authenticated data for an instance of GCM. -// This must be called before and data is encrypted. |key| must be the same -// value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on success -// and zero otherwise. -int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx, - const uint8_t *aad, size_t aad_len); - -// CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. |key| must be -// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on -// success and zero otherwise. -int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx, - const uint8_t *in, uint8_t *out, size_t len); - -// CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. |key| must be -// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on -// success and zero otherwise. -int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx, - const uint8_t *in, uint8_t *out, size_t len); - -// CRYPTO_gcm128_finish calculates the authenticator and compares it against -// |len| bytes of |tag|. |key| must be the same value that was passed to -// |CRYPTO_gcm128_init_ctx|. It returns one on success and zero otherwise. -int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx, - const uint8_t *tag, size_t len); - -// CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|. -// The minimum of |len| and 16 bytes are copied into |tag|. |key| must be the -// same value that was passed to |CRYPTO_gcm128_init_ctx|. -void CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx, uint8_t *tag, - size_t len); - - -// GCM assembly. - -void gcm_init_nohw(u128 Htable[16], const uint64_t H[2]); -void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, - size_t len); - -#if !defined(OPENSSL_NO_ASM) - -#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) -#define GCM_FUNCREF -void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]); -void gcm_gmult_clmul(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_clmul(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, - size_t len); - -void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]); -void gcm_gmult_ssse3(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_ssse3(uint8_t Xi[16], const u128 Htable[16], const uint8_t *in, - size_t len); - -#if defined(OPENSSL_X86_64) -#define GHASH_ASM_X86_64 -void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]); -void gcm_gmult_avx(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_avx(uint8_t Xi[16], const u128 Htable[16], const uint8_t *in, - size_t len); - -#define HW_GCM -size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - const u128 Htable[16], uint8_t Xi[16]); -size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - const u128 Htable[16], uint8_t Xi[16]); - -void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]); -void gcm_gmult_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16], - const uint8_t *in, size_t len); -void aes_gcm_enc_update_vaes_avx2(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, const uint8_t ivec[16], - const u128 Htable[16], uint8_t Xi[16]); -void aes_gcm_dec_update_vaes_avx2(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, const uint8_t ivec[16], - const u128 Htable[16], uint8_t Xi[16]); - -void gcm_init_vpclmulqdq_avx10_512(u128 Htable[16], const uint64_t H[2]); -void gcm_gmult_vpclmulqdq_avx10(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_vpclmulqdq_avx10_512(uint8_t Xi[16], const u128 Htable[16], - const uint8_t *in, size_t len); -void aes_gcm_enc_update_vaes_avx10_512(const uint8_t *in, uint8_t *out, - size_t len, const AES_KEY *key, - const uint8_t ivec[16], - const u128 Htable[16], uint8_t Xi[16]); -void aes_gcm_dec_update_vaes_avx10_512(const uint8_t *in, uint8_t *out, - size_t len, const AES_KEY *key, - const uint8_t ivec[16], - const u128 Htable[16], uint8_t Xi[16]); - -#endif // OPENSSL_X86_64 - -#if defined(OPENSSL_X86) -#define GHASH_ASM_X86 -#endif // OPENSSL_X86 - -#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) - -#define GHASH_ASM_ARM -#define GCM_FUNCREF - -inline int gcm_pmull_capable(void) { return CRYPTO_is_ARMv8_PMULL_capable(); } - -void gcm_init_v8(u128 Htable[16], const uint64_t H[2]); -void gcm_gmult_v8(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_v8(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, - size_t len); - -inline int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); } - -void gcm_init_neon(u128 Htable[16], const uint64_t H[2]); -void gcm_gmult_neon(uint8_t Xi[16], const u128 Htable[16]); -void gcm_ghash_neon(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, - size_t len); - -#if defined(OPENSSL_AARCH64) -#define HW_GCM -// These functions are defined in aesv8-gcm-armv8.pl. -void aes_gcm_enc_kernel(const uint8_t *in, uint64_t in_bits, void *out, - void *Xi, uint8_t *ivec, const AES_KEY *key, - const u128 Htable[16]); -void aes_gcm_dec_kernel(const uint8_t *in, uint64_t in_bits, void *out, - void *Xi, uint8_t *ivec, const AES_KEY *key, - const u128 Htable[16]); -#endif - -#endif -#endif // OPENSSL_NO_ASM - - -// CBC. - -// cbc128_f is the type of a function that performs CBC-mode encryption. -typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], int enc); - -// CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the -// given IV and block cipher in CBC mode. The input need not be a multiple of -// 128 bits long, but the output will round up to the nearest 128 bit multiple, -// zero padding the input if needed. The IV will be updated on return. -void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - block128_f block); - -// CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the -// given IV and block cipher in CBC mode. If |len| is not a multiple of 128 -// bits then only that many bytes will be written, but a multiple of 128 bits -// is always read from |in|. The IV will be updated on return. -void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - block128_f block); - - -// OFB. - -// CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode) -// |len| bytes from |in| to |out| using |block| in OFB mode. There's no -// requirement that |len| be a multiple of any value and any partial blocks are -// stored in |ivec| and |*num|, the latter must be zero before the initial -// call. -void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], unsigned *num, - block128_f block); - - -// CFB. - -// CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes -// from |in| to |out| using |block| in CFB mode. There's no requirement that -// |len| be a multiple of any value and any partial blocks are stored in |ivec| -// and |*num|, the latter must be zero before the initial call. -void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], unsigned *num, - int enc, block128_f block); - -// CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes -// from |in| to |out| using |block| in CFB-8 mode. Prior to the first call -// |num| should be set to zero. -void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - unsigned *num, int enc, block128_f block); - -// CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes -// from |in| to |out| using |block| in CFB-1 mode. Prior to the first call -// |num| should be set to zero. -void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits, - const AES_KEY *key, uint8_t ivec[16], - unsigned *num, int enc, block128_f block); - -size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t ivec[16], - block128_f block); - - -// POLYVAL. -// -// POLYVAL is a polynomial authenticator that operates over a field very -// similar to the one that GHASH uses. See -// https://www.rfc-editor.org/rfc/rfc8452.html#section-3. - -struct polyval_ctx { - uint8_t S[16]; - u128 Htable[16]; - gmult_func gmult; - ghash_func ghash; -}; - -// CRYPTO_POLYVAL_init initialises |ctx| using |key|. -void CRYPTO_POLYVAL_init(struct polyval_ctx *ctx, const uint8_t key[16]); - -// CRYPTO_POLYVAL_update_blocks updates the accumulator in |ctx| given the -// blocks from |in|. Only a whole number of blocks can be processed so |in_len| -// must be a multiple of 16. -void CRYPTO_POLYVAL_update_blocks(struct polyval_ctx *ctx, const uint8_t *in, - size_t in_len); - -// CRYPTO_POLYVAL_finish writes the accumulator from |ctx| to |out|. -void CRYPTO_POLYVAL_finish(const struct polyval_ctx *ctx, uint8_t out[16]); - - -#if defined(__cplusplus) -} // extern C -#endif - -#endif // OPENSSL_HEADER_MODES_INTERNAL_H
diff --git a/crypto/fipsmodule/rand/internal.h b/crypto/fipsmodule/rand/internal.h index ce2f63c..3c66438 100644 --- a/crypto/fipsmodule/rand/internal.h +++ b/crypto/fipsmodule/rand/internal.h
@@ -19,7 +19,7 @@ #include <openssl/ctrdrbg.h> #include "../../bcm_support.h" -#include "../modes/internal.h" +#include "../aes/internal.h" #if defined(__cplusplus) extern "C" {
diff --git a/decrepit/xts/xts.cc b/decrepit/xts/xts.cc index 79b1121..866880d 100644 --- a/decrepit/xts/xts.cc +++ b/decrepit/xts/xts.cc
@@ -15,7 +15,7 @@ #include <openssl/cipher.h> #include "../../crypto/fipsmodule/cipher/internal.h" -#include "../../crypto/fipsmodule/modes/internal.h" +#include "../../crypto/fipsmodule/aes/internal.h" typedef struct xts128_context {
diff --git a/gen/sources.bzl b/gen/sources.bzl index 45f41dd6..25ce8a1 100644 --- a/gen/sources.bzl +++ b/gen/sources.bzl
@@ -21,8 +21,15 @@ bcm_internal_headers = [ "crypto/fipsmodule/aes/aes.cc.inc", "crypto/fipsmodule/aes/aes_nohw.cc.inc", + "crypto/fipsmodule/aes/cbc.cc.inc", + "crypto/fipsmodule/aes/cfb.cc.inc", + "crypto/fipsmodule/aes/ctr.cc.inc", + "crypto/fipsmodule/aes/gcm.cc.inc", + "crypto/fipsmodule/aes/gcm_nohw.cc.inc", "crypto/fipsmodule/aes/key_wrap.cc.inc", "crypto/fipsmodule/aes/mode_wrappers.cc.inc", + "crypto/fipsmodule/aes/ofb.cc.inc", + "crypto/fipsmodule/aes/polyval.cc.inc", "crypto/fipsmodule/bn/add.cc.inc", "crypto/fipsmodule/bn/asm/x86_64-gcc.cc.inc", "crypto/fipsmodule/bn/bn.cc.inc", @@ -74,13 +81,6 @@ "crypto/fipsmodule/keccak/keccak.cc.inc", "crypto/fipsmodule/mldsa/mldsa.cc.inc", "crypto/fipsmodule/mlkem/mlkem.cc.inc", - "crypto/fipsmodule/modes/cbc.cc.inc", - "crypto/fipsmodule/modes/cfb.cc.inc", - "crypto/fipsmodule/modes/ctr.cc.inc", - "crypto/fipsmodule/modes/gcm.cc.inc", - "crypto/fipsmodule/modes/gcm_nohw.cc.inc", - "crypto/fipsmodule/modes/ofb.cc.inc", - "crypto/fipsmodule/modes/polyval.cc.inc", "crypto/fipsmodule/rand/ctrdrbg.cc.inc", "crypto/fipsmodule/rand/rand.cc.inc", "crypto/fipsmodule/rsa/blinding.cc.inc", @@ -619,7 +619,6 @@ "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/keccak/internal.h", - "crypto/fipsmodule/modes/internal.h", "crypto/fipsmodule/rand/internal.h", "crypto/fipsmodule/rsa/internal.h", "crypto/fipsmodule/service_indicator/internal.h", @@ -723,6 +722,7 @@ "crypto/evp/pbkdf_test.cc", "crypto/evp/scrypt_test.cc", "crypto/fipsmodule/aes/aes_test.cc", + "crypto/fipsmodule/aes/gcm_test.cc", "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", @@ -731,7 +731,6 @@ "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/hkdf/hkdf_test.cc", "crypto/fipsmodule/keccak/keccak_test.cc", - "crypto/fipsmodule/modes/gcm_test.cc", "crypto/fipsmodule/rand/ctrdrbg_test.cc", "crypto/fipsmodule/service_indicator/service_indicator_test.cc", "crypto/fipsmodule/sha/sha_test.cc",
diff --git a/gen/sources.cmake b/gen/sources.cmake index dec16ac..282211d 100644 --- a/gen/sources.cmake +++ b/gen/sources.cmake
@@ -25,8 +25,15 @@ crypto/fipsmodule/aes/aes.cc.inc crypto/fipsmodule/aes/aes_nohw.cc.inc + crypto/fipsmodule/aes/cbc.cc.inc + crypto/fipsmodule/aes/cfb.cc.inc + crypto/fipsmodule/aes/ctr.cc.inc + crypto/fipsmodule/aes/gcm.cc.inc + crypto/fipsmodule/aes/gcm_nohw.cc.inc crypto/fipsmodule/aes/key_wrap.cc.inc crypto/fipsmodule/aes/mode_wrappers.cc.inc + crypto/fipsmodule/aes/ofb.cc.inc + crypto/fipsmodule/aes/polyval.cc.inc crypto/fipsmodule/bn/add.cc.inc crypto/fipsmodule/bn/asm/x86_64-gcc.cc.inc crypto/fipsmodule/bn/bn.cc.inc @@ -78,13 +85,6 @@ crypto/fipsmodule/keccak/keccak.cc.inc crypto/fipsmodule/mldsa/mldsa.cc.inc crypto/fipsmodule/mlkem/mlkem.cc.inc - crypto/fipsmodule/modes/cbc.cc.inc - crypto/fipsmodule/modes/cfb.cc.inc - crypto/fipsmodule/modes/ctr.cc.inc - crypto/fipsmodule/modes/gcm.cc.inc - crypto/fipsmodule/modes/gcm_nohw.cc.inc - crypto/fipsmodule/modes/ofb.cc.inc - crypto/fipsmodule/modes/polyval.cc.inc crypto/fipsmodule/rand/ctrdrbg.cc.inc crypto/fipsmodule/rand/rand.cc.inc crypto/fipsmodule/rsa/blinding.cc.inc @@ -637,7 +637,6 @@ crypto/fipsmodule/ec/p256_table.h crypto/fipsmodule/ecdsa/internal.h crypto/fipsmodule/keccak/internal.h - crypto/fipsmodule/modes/internal.h crypto/fipsmodule/rand/internal.h crypto/fipsmodule/rsa/internal.h crypto/fipsmodule/service_indicator/internal.h @@ -747,6 +746,7 @@ crypto/evp/pbkdf_test.cc crypto/evp/scrypt_test.cc crypto/fipsmodule/aes/aes_test.cc + crypto/fipsmodule/aes/gcm_test.cc crypto/fipsmodule/bn/bn_test.cc crypto/fipsmodule/cmac/cmac_test.cc crypto/fipsmodule/ec/ec_test.cc @@ -755,7 +755,6 @@ crypto/fipsmodule/ecdsa/ecdsa_test.cc crypto/fipsmodule/hkdf/hkdf_test.cc crypto/fipsmodule/keccak/keccak_test.cc - crypto/fipsmodule/modes/gcm_test.cc crypto/fipsmodule/rand/ctrdrbg_test.cc crypto/fipsmodule/service_indicator/service_indicator_test.cc crypto/fipsmodule/sha/sha_test.cc
diff --git a/gen/sources.gni b/gen/sources.gni index 9461237..ad08afb 100644 --- a/gen/sources.gni +++ b/gen/sources.gni
@@ -21,8 +21,15 @@ bcm_internal_headers = [ "crypto/fipsmodule/aes/aes.cc.inc", "crypto/fipsmodule/aes/aes_nohw.cc.inc", + "crypto/fipsmodule/aes/cbc.cc.inc", + "crypto/fipsmodule/aes/cfb.cc.inc", + "crypto/fipsmodule/aes/ctr.cc.inc", + "crypto/fipsmodule/aes/gcm.cc.inc", + "crypto/fipsmodule/aes/gcm_nohw.cc.inc", "crypto/fipsmodule/aes/key_wrap.cc.inc", "crypto/fipsmodule/aes/mode_wrappers.cc.inc", + "crypto/fipsmodule/aes/ofb.cc.inc", + "crypto/fipsmodule/aes/polyval.cc.inc", "crypto/fipsmodule/bn/add.cc.inc", "crypto/fipsmodule/bn/asm/x86_64-gcc.cc.inc", "crypto/fipsmodule/bn/bn.cc.inc", @@ -74,13 +81,6 @@ "crypto/fipsmodule/keccak/keccak.cc.inc", "crypto/fipsmodule/mldsa/mldsa.cc.inc", "crypto/fipsmodule/mlkem/mlkem.cc.inc", - "crypto/fipsmodule/modes/cbc.cc.inc", - "crypto/fipsmodule/modes/cfb.cc.inc", - "crypto/fipsmodule/modes/ctr.cc.inc", - "crypto/fipsmodule/modes/gcm.cc.inc", - "crypto/fipsmodule/modes/gcm_nohw.cc.inc", - "crypto/fipsmodule/modes/ofb.cc.inc", - "crypto/fipsmodule/modes/polyval.cc.inc", "crypto/fipsmodule/rand/ctrdrbg.cc.inc", "crypto/fipsmodule/rand/rand.cc.inc", "crypto/fipsmodule/rsa/blinding.cc.inc", @@ -619,7 +619,6 @@ "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/keccak/internal.h", - "crypto/fipsmodule/modes/internal.h", "crypto/fipsmodule/rand/internal.h", "crypto/fipsmodule/rsa/internal.h", "crypto/fipsmodule/service_indicator/internal.h", @@ -723,6 +722,7 @@ "crypto/evp/pbkdf_test.cc", "crypto/evp/scrypt_test.cc", "crypto/fipsmodule/aes/aes_test.cc", + "crypto/fipsmodule/aes/gcm_test.cc", "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", @@ -731,7 +731,6 @@ "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/hkdf/hkdf_test.cc", "crypto/fipsmodule/keccak/keccak_test.cc", - "crypto/fipsmodule/modes/gcm_test.cc", "crypto/fipsmodule/rand/ctrdrbg_test.cc", "crypto/fipsmodule/service_indicator/service_indicator_test.cc", "crypto/fipsmodule/sha/sha_test.cc",
diff --git a/gen/sources.json b/gen/sources.json index a37daf1..6f2e84f 100644 --- a/gen/sources.json +++ b/gen/sources.json
@@ -6,8 +6,15 @@ "internal_hdrs": [ "crypto/fipsmodule/aes/aes.cc.inc", "crypto/fipsmodule/aes/aes_nohw.cc.inc", + "crypto/fipsmodule/aes/cbc.cc.inc", + "crypto/fipsmodule/aes/cfb.cc.inc", + "crypto/fipsmodule/aes/ctr.cc.inc", + "crypto/fipsmodule/aes/gcm.cc.inc", + "crypto/fipsmodule/aes/gcm_nohw.cc.inc", "crypto/fipsmodule/aes/key_wrap.cc.inc", "crypto/fipsmodule/aes/mode_wrappers.cc.inc", + "crypto/fipsmodule/aes/ofb.cc.inc", + "crypto/fipsmodule/aes/polyval.cc.inc", "crypto/fipsmodule/bn/add.cc.inc", "crypto/fipsmodule/bn/asm/x86_64-gcc.cc.inc", "crypto/fipsmodule/bn/bn.cc.inc", @@ -59,13 +66,6 @@ "crypto/fipsmodule/keccak/keccak.cc.inc", "crypto/fipsmodule/mldsa/mldsa.cc.inc", "crypto/fipsmodule/mlkem/mlkem.cc.inc", - "crypto/fipsmodule/modes/cbc.cc.inc", - "crypto/fipsmodule/modes/cfb.cc.inc", - "crypto/fipsmodule/modes/ctr.cc.inc", - "crypto/fipsmodule/modes/gcm.cc.inc", - "crypto/fipsmodule/modes/gcm_nohw.cc.inc", - "crypto/fipsmodule/modes/ofb.cc.inc", - "crypto/fipsmodule/modes/polyval.cc.inc", "crypto/fipsmodule/rand/ctrdrbg.cc.inc", "crypto/fipsmodule/rand/rand.cc.inc", "crypto/fipsmodule/rsa/blinding.cc.inc", @@ -601,7 +601,6 @@ "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/keccak/internal.h", - "crypto/fipsmodule/modes/internal.h", "crypto/fipsmodule/rand/internal.h", "crypto/fipsmodule/rsa/internal.h", "crypto/fipsmodule/service_indicator/internal.h", @@ -704,6 +703,7 @@ "crypto/evp/pbkdf_test.cc", "crypto/evp/scrypt_test.cc", "crypto/fipsmodule/aes/aes_test.cc", + "crypto/fipsmodule/aes/gcm_test.cc", "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", @@ -712,7 +712,6 @@ "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/hkdf/hkdf_test.cc", "crypto/fipsmodule/keccak/keccak_test.cc", - "crypto/fipsmodule/modes/gcm_test.cc", "crypto/fipsmodule/rand/ctrdrbg_test.cc", "crypto/fipsmodule/service_indicator/service_indicator_test.cc", "crypto/fipsmodule/sha/sha_test.cc",