Always define GHASH.
There is a C implementation of gcm_ghash_4bit to pair with
gcm_gmult_4bit. It's even slightly faster per the numbers below (x86_64
OPENSSL_NO_ASM build), but, more importantly, we trim down the
combinatorial explosion of GCM implementations and free up complexity
budget for potentially using bsaes better in the future.
Old:
Did 2557000 AES-128-GCM (16 bytes) seal operations in 1000057us (2556854.3 ops/sec): 40.9 MB/s
Did 94000 AES-128-GCM (1350 bytes) seal operations in 1009613us (93105.0 ops/sec): 125.7 MB/s
Did 17000 AES-128-GCM (8192 bytes) seal operations in 1024768us (16589.1 ops/sec): 135.9 MB/s
Did 2511000 AES-256-GCM (16 bytes) seal operations in 1000196us (2510507.9 ops/sec): 40.2 MB/s
Did 84000 AES-256-GCM (1350 bytes) seal operations in 1000412us (83965.4 ops/sec): 113.4 MB/s
Did 15000 AES-256-GCM (8192 bytes) seal operations in 1046963us (14327.2 ops/sec): 117.4 MB/s
New:
Did 2739000 AES-128-GCM (16 bytes) seal operations in 1000322us (2738118.3 ops/sec): 43.8 MB/s
Did 100000 AES-128-GCM (1350 bytes) seal operations in 1008190us (99187.7 ops/sec): 133.9 MB/s
Did 17000 AES-128-GCM (8192 bytes) seal operations in 1006360us (16892.6 ops/sec): 138.4 MB/s
Did 2546000 AES-256-GCM (16 bytes) seal operations in 1000150us (2545618.2 ops/sec): 40.7 MB/s
Did 86000 AES-256-GCM (1350 bytes) seal operations in 1000970us (85916.7 ops/sec): 116.0 MB/s
Did 14850 AES-256-GCM (8192 bytes) seal operations in 1023459us (14509.6 ops/sec): 118.9 MB/s
While I'm here, tighten up some of the functions and align the ctr32 and
non-ctr32 paths.
Bug: 256
Change-Id: Id4df699cefc8630dd5a350d44f927900340f5e60
Reviewed-on: https://boringssl-review.googlesource.com/c/34869
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/fipsmodule/modes/gcm.c b/crypto/fipsmodule/modes/gcm.c
index 681f7a9..97fde3e 100644
--- a/crypto/fipsmodule/modes/gcm.c
+++ b/crypto/fipsmodule/modes/gcm.c
@@ -234,14 +234,12 @@
#endif // !GHASH_ASM || AARCH64 || PPC64LE
#define GCM_MUL(ctx, Xi) gcm_gmult_4bit((ctx)->Xi.u, (ctx)->gcm_key.Htable)
-#if defined(GHASH_ASM)
#define GHASH(ctx, in, len) \
gcm_ghash_4bit((ctx)->Xi.u, (ctx)->gcm_key.Htable, in, len)
// GHASH_CHUNK is "stride parameter" missioned to mitigate cache
// trashing effect. In other words idea is to hash data while it's
// still in L1 cache after encryption pass...
#define GHASH_CHUNK (3 * 1024)
-#endif // GHASH_ASM
#if defined(GHASH_ASM_X86_64)
void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]) {
@@ -273,11 +271,9 @@
#ifdef GCM_FUNCREF_4BIT
#undef GCM_MUL
#define GCM_MUL(ctx, Xi) (*gcm_gmult_p)((ctx)->Xi.u, (ctx)->gcm_key.Htable)
-#ifdef GHASH
#undef GHASH
#define GHASH(ctx, in, len) \
(*gcm_ghash_p)((ctx)->Xi.u, (ctx)->gcm_key.Htable, in, len)
-#endif
#endif // GCM_FUNCREF_4BIT
void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
@@ -376,7 +372,6 @@
void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key,
const uint8_t *iv, size_t len) {
- unsigned int ctr;
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
@@ -391,6 +386,7 @@
ctx->ares = 0;
ctx->mres = 0;
+ uint32_t ctr;
if (len == 12) {
OPENSSL_memcpy(ctx->Yi.c, iv, 12);
ctx->Yi.c[15] = 1;
@@ -425,8 +421,6 @@
}
int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, size_t len) {
- unsigned int n;
- uint64_t alen = ctx->len.u[0];
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
@@ -440,13 +434,13 @@
return 0;
}
- alen += len;
+ uint64_t alen = ctx->len.u[0] + len;
if (alen > (UINT64_C(1) << 61) || (sizeof(len) == 8 && alen < len)) {
return 0;
}
ctx->len.u[0] = alen;
- n = ctx->ares;
+ unsigned n = ctx->ares;
if (n) {
while (n && len) {
ctx->Xi.c[n] ^= *(aad++);
@@ -462,23 +456,12 @@
}
// Process a whole number of blocks.
-#ifdef GHASH
size_t len_blocks = len & kSizeTWithoutLower4Bits;
if (len_blocks != 0) {
GHASH(ctx, aad, len_blocks);
aad += len_blocks;
len -= len_blocks;
}
-#else
- while (len >= 16) {
- for (size_t i = 0; i < 16; ++i) {
- ctx->Xi.c[i] ^= aad[i];
- }
- GCM_MUL(ctx, Xi);
- aad += 16;
- len -= 16;
- }
-#endif
// Process the remainder.
if (len != 0) {
@@ -494,19 +477,15 @@
int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
const uint8_t *in, uint8_t *out, size_t len) {
- unsigned int n, ctr;
- uint64_t mlen = ctx->len.u[1];
block128_f block = ctx->gcm_key.block;
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
-#ifdef GHASH
void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
size_t len) = ctx->gcm_key.ghash;
#endif
-#endif
- mlen += len;
+ uint64_t mlen = ctx->len.u[1] + len;
if (mlen > ((UINT64_C(1) << 36) - 32) ||
(sizeof(len) == 8 && mlen < len)) {
return 0;
@@ -519,9 +498,7 @@
ctx->ares = 0;
}
- ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
-
- n = ctx->mres;
+ unsigned n = ctx->mres;
if (n) {
while (n && len) {
ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
@@ -535,6 +512,8 @@
return 1;
}
}
+
+ uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
if (STRICT_ALIGNMENT &&
((uintptr_t)in | (uintptr_t)out) % sizeof(size_t) != 0) {
for (size_t i = 0; i < len; ++i) {
@@ -553,7 +532,6 @@
ctx->mres = n;
return 1;
}
-#if defined(GHASH) && defined(GHASH_CHUNK)
while (len >= GHASH_CHUNK) {
size_t j = GHASH_CHUNK;
@@ -588,22 +566,6 @@
}
GHASH(ctx, out - len_blocks, len_blocks);
}
-#else
- while (len >= 16) {
- (*block)(ctx->Yi.c, ctx->EKi.c, key);
- ++ctr;
- ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
- for (size_t i = 0; i < 16; i += sizeof(size_t)) {
- size_t tmp = load_word_le(in + i) ^ ctx->EKi.t[i / sizeof(size_t)];
- store_word_le(out + i, tmp);
- ctx->Xi.t[i / sizeof(size_t)] ^= tmp;
- }
- GCM_MUL(ctx, Xi);
- out += 16;
- in += 16;
- len -= 16;
- }
-#endif
if (len) {
(*block)(ctx->Yi.c, ctx->EKi.c, key);
++ctr;
@@ -621,19 +583,15 @@
int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key,
const unsigned char *in, unsigned char *out,
size_t len) {
- unsigned int n, ctr;
- uint64_t mlen = ctx->len.u[1];
block128_f block = ctx->gcm_key.block;
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
-#ifdef GHASH
void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
size_t len) = ctx->gcm_key.ghash;
#endif
-#endif
- mlen += len;
+ uint64_t mlen = ctx->len.u[1] + len;
if (mlen > ((UINT64_C(1) << 36) - 32) ||
(sizeof(len) == 8 && mlen < len)) {
return 0;
@@ -646,9 +604,7 @@
ctx->ares = 0;
}
- ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
-
- n = ctx->mres;
+ unsigned n = ctx->mres;
if (n) {
while (n && len) {
uint8_t c = *(in++);
@@ -664,6 +620,8 @@
return 1;
}
}
+
+ uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
if (STRICT_ALIGNMENT &&
((uintptr_t)in | (uintptr_t)out) % sizeof(size_t) != 0) {
for (size_t i = 0; i < len; ++i) {
@@ -685,7 +643,6 @@
ctx->mres = n;
return 1;
}
-#if defined(GHASH) && defined(GHASH_CHUNK)
while (len >= GHASH_CHUNK) {
size_t j = GHASH_CHUNK;
@@ -720,22 +677,6 @@
len -= 16;
}
}
-#else
- while (len >= 16) {
- (*block)(ctx->Yi.c, ctx->EKi.c, key);
- ++ctr;
- ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
- for (size_t i = 0; i < 16; i += sizeof(size_t)) {
- size_t c = load_word_le(in + i);
- store_word_le(out + i, c ^ ctx->EKi.t[i / sizeof(size_t)]);
- ctx->Xi.t[i / sizeof(size_t)] ^= c;
- }
- GCM_MUL(ctx, Xi);
- out += 16;
- in += 16;
- len -= 16;
- }
-#endif
if (len) {
(*block)(ctx->Yi.c, ctx->EKi.c, key);
++ctr;
@@ -755,18 +696,14 @@
int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
const uint8_t *in, uint8_t *out, size_t len,
ctr128_f stream) {
- unsigned int n, ctr;
- uint64_t mlen = ctx->len.u[1];
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
-#ifdef GHASH
void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
size_t len) = ctx->gcm_key.ghash;
#endif
-#endif
- mlen += len;
+ uint64_t mlen = ctx->len.u[1] + len;
if (mlen > ((UINT64_C(1) << 36) - 32) ||
(sizeof(len) == 8 && mlen < len)) {
return 0;
@@ -779,7 +716,7 @@
ctx->ares = 0;
}
- n = ctx->mres;
+ unsigned n = ctx->mres;
if (n) {
while (n && len) {
ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
@@ -805,9 +742,7 @@
}
#endif
- ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
-
-#if defined(GHASH)
+ uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
while (len >= GHASH_CHUNK) {
(*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
ctr += GHASH_CHUNK / 16;
@@ -817,28 +752,17 @@
in += GHASH_CHUNK;
len -= GHASH_CHUNK;
}
-#endif
- size_t i = len & kSizeTWithoutLower4Bits;
- if (i != 0) {
- size_t j = i / 16;
+ size_t len_blocks = len & kSizeTWithoutLower4Bits;
+ if (len_blocks != 0) {
+ size_t j = len_blocks / 16;
(*stream)(in, out, j, key, ctx->Yi.c);
ctr += (unsigned int)j;
ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
- in += i;
- len -= i;
-#if defined(GHASH)
- GHASH(ctx, out, i);
- out += i;
-#else
- while (j--) {
- for (i = 0; i < 16; ++i) {
- ctx->Xi.c[i] ^= out[i];
- }
- GCM_MUL(ctx, Xi);
- out += 16;
- }
-#endif
+ in += len_blocks;
+ len -= len_blocks;
+ GHASH(ctx, out, len_blocks);
+ out += len_blocks;
}
if (len) {
(*ctx->gcm_key.block)(ctx->Yi.c, ctx->EKi.c, key);
@@ -857,18 +781,14 @@
int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key,
const uint8_t *in, uint8_t *out, size_t len,
ctr128_f stream) {
- unsigned int n, ctr;
- uint64_t mlen = ctx->len.u[1];
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
-#ifdef GHASH
void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
size_t len) = ctx->gcm_key.ghash;
#endif
-#endif
- mlen += len;
+ uint64_t mlen = ctx->len.u[1] + len;
if (mlen > ((UINT64_C(1) << 36) - 32) ||
(sizeof(len) == 8 && mlen < len)) {
return 0;
@@ -881,7 +801,7 @@
ctx->ares = 0;
}
- n = ctx->mres;
+ unsigned n = ctx->mres;
if (n) {
while (n && len) {
uint8_t c = *(in++);
@@ -909,9 +829,7 @@
}
#endif
- ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
-
-#if defined(GHASH)
+ uint32_t ctr = CRYPTO_bswap4(ctx->Yi.d[3]);
while (len >= GHASH_CHUNK) {
GHASH(ctx, in, GHASH_CHUNK);
(*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
@@ -921,31 +839,17 @@
in += GHASH_CHUNK;
len -= GHASH_CHUNK;
}
-#endif
- size_t i = len & kSizeTWithoutLower4Bits;
- if (i != 0) {
- size_t j = i / 16;
+ size_t len_blocks = len & kSizeTWithoutLower4Bits;
+ if (len_blocks != 0) {
+ size_t j = len_blocks / 16;
-#if defined(GHASH)
- GHASH(ctx, in, i);
-#else
- while (j--) {
- size_t k;
- for (k = 0; k < 16; ++k) {
- ctx->Xi.c[k] ^= in[k];
- }
- GCM_MUL(ctx, Xi);
- in += 16;
- }
- j = i / 16;
- in -= i;
-#endif
+ GHASH(ctx, in, len_blocks);
(*stream)(in, out, j, key, ctx->Yi.c);
ctr += (unsigned int)j;
ctx->Yi.d[3] = CRYPTO_bswap4(ctr);
- out += i;
- in += i;
- len -= i;
+ out += len_blocks;
+ in += len_blocks;
+ len -= len_blocks;
}
if (len) {
(*ctx->gcm_key.block)(ctx->Yi.c, ctx->EKi.c, key);
@@ -964,8 +868,6 @@
}
int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) {
- uint64_t alen = ctx->len.u[0] << 3;
- uint64_t clen = ctx->len.u[1] << 3;
#ifdef GCM_FUNCREF_4BIT
void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) =
ctx->gcm_key.gmult;
@@ -975,11 +877,8 @@
GCM_MUL(ctx, Xi);
}
- alen = CRYPTO_bswap8(alen);
- clen = CRYPTO_bswap8(clen);
-
- ctx->Xi.u[0] ^= alen;
- ctx->Xi.u[1] ^= clen;
+ ctx->Xi.u[0] ^= CRYPTO_bswap8(ctx->len.u[0] << 3);
+ ctx->Xi.u[1] ^= CRYPTO_bswap8(ctx->len.u[1] << 3);
GCM_MUL(ctx, Xi);
ctx->Xi.u[0] ^= ctx->EK0.u[0];
