AEAD benchmarks: add iovec variants. Bug: 542671322 Change-Id: Ia020f35b4be914bc8d96f328082e566b6a6a6964 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/100587 Commit-Queue: Rudolf Polzer <rpolzer@google.com> Reviewed-by: Xiangfei Ding <xfding@google.com> Presubmit-BoringSSL-Verified: boringssl-scoped@luci-project-accounts.iam.gserviceaccount.com <boringssl-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/bench/aead.cc b/bench/aead.cc index dd90f1e..bbdfaed 100644 --- a/bench/aead.cc +++ b/bench/aead.cc
@@ -132,17 +132,148 @@ } } +void BM_SpeedAEADv(benchmark::State &state, size_t ad_len, + evp_aead_direction_t direction, const EVP_AEAD *aead) { + const unsigned kAlignment = 16; + const size_t input_len = static_cast<size_t>(state.range(0)); + const size_t iovec_first = static_cast<size_t>(state.range(1)); + const size_t iovec_others = static_cast<size_t>(state.range(2)); + ScopedEVP_AEAD_CTX ctx; + const size_t key_len = EVP_AEAD_key_length(aead); + const size_t nonce_len = EVP_AEAD_nonce_length(aead); + const size_t overhead_len = EVP_AEAD_max_overhead(aead); + + std::vector<uint8_t> key(key_len); + std::vector<uint8_t> nonce(nonce_len); + std::vector<uint8_t> in_storage(input_len + kAlignment); + // N.B. for EVP_AEAD_CTX_sealv the input and output buffers may be the + // same size. However, in the direction == evp_aead_open case we copy the tag + // to the end of the output to use the non-gathering openv (and to minimize + // total chunk count), hence we add overhead_len to the size of this buffer. + std::vector<uint8_t> out_storage(input_len + overhead_len + kAlignment); + std::vector<uint8_t> in2_storage(input_len + overhead_len + kAlignment); + std::vector<uint8_t> ad(ad_len); + std::vector<uint8_t> tag_storage(overhead_len + kAlignment); + + uint8_t *in = + static_cast<uint8_t *>(align_pointer(in_storage.data(), kAlignment)); + uint8_t *out = + static_cast<uint8_t *>(align_pointer(out_storage.data(), kAlignment)); + uint8_t *tag = + static_cast<uint8_t *>(align_pointer(tag_storage.data(), kAlignment)); + uint8_t *in2 = + static_cast<uint8_t *>(align_pointer(in2_storage.data(), kAlignment)); + + if (!EVP_AEAD_CTX_init_with_direction(ctx.get(), aead, key.data(), key_len, + EVP_AEAD_DEFAULT_TAG_LENGTH, + evp_aead_seal)) { + state.SkipWithError("Failed to create EVP_AEAD_CTX."); + return; + } + + // If "first" is unaligned, unalign all pointers by one. As kAlignment and + // not just kAlignment-1 was added in the allocation, this always fits. + if (iovec_first % kAlignment) { + ++in; + ++out; + ++tag; + ++in2; + } + + std::vector<CRYPTO_IOVEC> inout_vec; + std::vector<CRYPTO_IOVEC> outin2_vec; + for (size_t pos = 0; pos < input_len;) { + size_t can = (pos == 0) ? iovec_first : iovec_others; + size_t have = input_len - pos; + size_t take = std::min(can, have); + inout_vec.push_back(CRYPTO_IOVEC{out + pos, in + pos, take}); + outin2_vec.push_back(CRYPTO_IOVEC{in2 + pos, out + pos, take}); + pos += take; + } + CRYPTO_IVEC ad_vec = {ad.data(), ad_len}; + + if (direction == evp_aead_seal) { + size_t tag_len; + for (auto _ : state) { + benchmark::DoNotOptimize(inout_vec); + benchmark::DoNotOptimize(nonce.data()); + benchmark::DoNotOptimize(ad_vec); + if (!EVP_AEAD_CTX_sealv(ctx.get(), inout_vec.data(), inout_vec.size(), + tag, &tag_len, overhead_len, nonce.data(), + nonce_len, &ad_vec, 1)) { + state.SkipWithError("EVP_AEAD_CTX_sealv failed."); + return; + } + benchmark::DoNotOptimize(out); + benchmark::DoNotOptimize(tag); + benchmark::DoNotOptimize(tag_len); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * input_len); + } else { + size_t tag_len; + if (!EVP_AEAD_CTX_sealv(ctx.get(), inout_vec.data(), inout_vec.size(), tag, + &tag_len, overhead_len, nonce.data(), nonce_len, + &ad_vec, 1)) { + state.SkipWithError("EVP_AEAD_CTX_sealv failed."); + return; + } + + ctx.Reset(); + if (!EVP_AEAD_CTX_init_with_direction(ctx.get(), aead, key.data(), key_len, + EVP_AEAD_DEFAULT_TAG_LENGTH, + evp_aead_open)) { + state.SkipWithError("Failed to create EVP_AEAD_CTX."); + return; + } + + // We know `out` and `in2` have enough space for the tag; so let's append + // it to the last chunk of `out`. + OPENSSL_memcpy(inout_vec.back().out + inout_vec.back().len, tag, tag_len); + outin2_vec.back().len += tag_len; + + size_t in2_len; + for (auto _ : state) { + benchmark::DoNotOptimize(outin2_vec); + benchmark::DoNotOptimize(nonce.data()); + benchmark::DoNotOptimize(ad_vec); + // N.B. EVP_AEAD_CTX_openv_detached is not implemented for all AEADs. + if (!EVP_AEAD_CTX_openv(ctx.get(), outin2_vec.data(), outin2_vec.size(), + &in2_len, nonce.data(), nonce_len, &ad_vec, 1)) { + state.SkipWithError("EVP_AEAD_CTX_openv failed."); + return; + } + benchmark::DoNotOptimize(in2); + benchmark::DoNotOptimize(in2_len); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * input_len); + } +} + static const int64_t kInputSizes[] = {16, 256, 1350, 8192, 16384}; void SetInputLength(benchmark::Benchmark *bench) { - bench->ArgName("InputSize"); + bench->ArgNames({"InputSize"}); auto input_sizes = bench::GetInputSizes(bench); - if (input_sizes.empty()) { - bench->ArgsProduct( - {std::vector<int64_t>(kInputSizes, std::end(kInputSizes))}); - } else { - bench->ArgsProduct( - {std::vector<int64_t>(input_sizes.begin(), input_sizes.end())}); + for (const int64_t size : input_sizes.empty() ? kInputSizes : input_sizes) { + bench->Args({size}); + } +} + +void SetInputLengthv(benchmark::Benchmark *bench) { + // No need to distinguish by name - the IOVec related args suffice to + // distinguish them. + bench->Name("BM_SpeedAEAD"); + + bench->ArgNames({"InputSize", "IOVecFirst", "IOVecOthers"}); + auto input_sizes = bench::GetInputSizes(bench); + for (const int64_t size : input_sizes.empty() ? kInputSizes : input_sizes) { + bench->Args({size, size, 0}); // One shot. + if (size >= 1456) { + bench->Args({size, 1456, 1456}); // Nicely aligned. + bench->Args({size, 1457, 1456}); // As unaligned as it gets. + } } } @@ -153,6 +284,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_gcm, kTLSADLen, evp_aead_open, EVP_aead_aes_128_gcm()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_gcm, kTLSADLen, evp_aead_seal, + EVP_aead_aes_128_gcm()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_gcm, kTLSADLen, evp_aead_open, + EVP_aead_aes_128_gcm()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_192_gcm, kTLSADLen, evp_aead_seal, EVP_aead_aes_192_gcm()) @@ -160,6 +297,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_192_gcm, kTLSADLen, evp_aead_open, EVP_aead_aes_192_gcm()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_192_gcm, kTLSADLen, evp_aead_seal, + EVP_aead_aes_192_gcm()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_192_gcm, kTLSADLen, evp_aead_open, + EVP_aead_aes_192_gcm()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_256_gcm, kTLSADLen, evp_aead_seal, EVP_aead_aes_256_gcm()) @@ -167,6 +310,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_256_gcm, kTLSADLen, evp_aead_open, EVP_aead_aes_256_gcm()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_256_gcm, kTLSADLen, evp_aead_seal, + EVP_aead_aes_256_gcm()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_256_gcm, kTLSADLen, evp_aead_open, + EVP_aead_aes_256_gcm()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_chacha20_poly1305, kTLSADLen, evp_aead_seal, EVP_aead_chacha20_poly1305()) @@ -174,6 +323,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_chacha20_poly1305, kTLSADLen, evp_aead_open, EVP_aead_chacha20_poly1305()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_chacha20_poly1305, kTLSADLen, + evp_aead_seal, EVP_aead_chacha20_poly1305()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_chacha20_poly1305, kTLSADLen, + evp_aead_open, EVP_aead_chacha20_poly1305()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_xchacha20_poly1305, kTLSADLen, evp_aead_seal, EVP_aead_xchacha20_poly1305()) @@ -181,6 +336,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_xchacha20_poly1305, kTLSADLen, evp_aead_open, EVP_aead_xchacha20_poly1305()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_xchacha20_poly1305, kTLSADLen, + evp_aead_seal, EVP_aead_xchacha20_poly1305()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_xchacha20_poly1305, kTLSADLen, + evp_aead_open, EVP_aead_xchacha20_poly1305()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_cbc_sha1, kLegacyADLen, evp_aead_seal, EVP_aead_aes_128_cbc_sha1_tls()) @@ -188,6 +349,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_cbc_sha1, kLegacyADLen, evp_aead_open, EVP_aead_aes_128_cbc_sha1_tls()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_cbc_sha1, kLegacyADLen, + evp_aead_seal, EVP_aead_aes_128_cbc_sha1_tls()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_cbc_sha1, kLegacyADLen, + evp_aead_open, EVP_aead_aes_128_cbc_sha1_tls()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_cbc_sha256, kLegacyADLen, evp_aead_seal, EVP_aead_aes_128_cbc_sha256_tls()) @@ -195,6 +362,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_cbc_sha256, kLegacyADLen, evp_aead_open, EVP_aead_aes_128_cbc_sha256_tls()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_cbc_sha256, kLegacyADLen, + evp_aead_seal, EVP_aead_aes_128_cbc_sha256_tls()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_cbc_sha256, kLegacyADLen, + evp_aead_open, EVP_aead_aes_128_cbc_sha256_tls()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_256_cbc_sha1, kLegacyADLen, evp_aead_seal, EVP_aead_aes_256_cbc_sha1_tls()) @@ -202,6 +375,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_256_cbc_sha1, kLegacyADLen, evp_aead_open, EVP_aead_aes_256_cbc_sha1_tls()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_256_cbc_sha1, kLegacyADLen, + evp_aead_seal, EVP_aead_aes_256_cbc_sha1_tls()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_256_cbc_sha1, kLegacyADLen, + evp_aead_open, EVP_aead_aes_256_cbc_sha1_tls()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_gcm_siv, kTLSADLen, evp_aead_seal, EVP_aead_aes_128_gcm_siv()) @@ -209,6 +388,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_gcm_siv, kTLSADLen, evp_aead_open, EVP_aead_aes_128_gcm_siv()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_gcm_siv, kTLSADLen, + evp_aead_seal, EVP_aead_aes_128_gcm_siv()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_gcm_siv, kTLSADLen, + evp_aead_open, EVP_aead_aes_128_gcm_siv()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_256_gcm_siv, kTLSADLen, evp_aead_seal, EVP_aead_aes_256_gcm_siv()) @@ -216,6 +401,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_256_gcm_siv, kTLSADLen, evp_aead_open, EVP_aead_aes_256_gcm_siv()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_256_gcm_siv, kTLSADLen, + evp_aead_seal, EVP_aead_aes_256_gcm_siv()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_256_gcm_siv, kTLSADLen, + evp_aead_open, EVP_aead_aes_256_gcm_siv()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_eax, kTLSADLen, evp_aead_seal, EVP_aead_aes_128_eax()) @@ -223,6 +414,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_eax, kTLSADLen, evp_aead_open, EVP_aead_aes_128_eax()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_eax, kTLSADLen, evp_aead_seal, + EVP_aead_aes_128_eax()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_eax, kTLSADLen, evp_aead_open, + EVP_aead_aes_128_eax()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_256_eax, kTLSADLen, evp_aead_seal, EVP_aead_aes_256_eax()) @@ -230,6 +427,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_256_eax, kTLSADLen, evp_aead_open, EVP_aead_aes_256_eax()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_256_eax, kTLSADLen, evp_aead_seal, + EVP_aead_aes_256_eax()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_256_eax, kTLSADLen, evp_aead_open, + EVP_aead_aes_256_eax()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_ccm_bluetooth, kTLSADLen, evp_aead_seal, EVP_aead_aes_128_ccm_bluetooth()) @@ -237,6 +440,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_ccm_bluetooth, kTLSADLen, evp_aead_open, EVP_aead_aes_128_ccm_bluetooth()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_ccm_bluetooth, kTLSADLen, + evp_aead_seal, EVP_aead_aes_128_ccm_bluetooth()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_ccm_bluetooth, kTLSADLen, + evp_aead_open, EVP_aead_aes_128_ccm_bluetooth()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_ccm_bluetooth8, kTLSADLen, evp_aead_seal, EVP_aead_aes_128_ccm_bluetooth_8()) @@ -244,6 +453,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_ccm_bluetooth8, kTLSADLen, evp_aead_open, EVP_aead_aes_128_ccm_bluetooth_8()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_ccm_bluetooth8, kTLSADLen, + evp_aead_seal, EVP_aead_aes_128_ccm_bluetooth_8()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_ccm_bluetooth8, kTLSADLen, + evp_aead_open, EVP_aead_aes_128_ccm_bluetooth_8()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_ccm_matter, kTLSADLen, evp_aead_seal, EVP_aead_aes_128_ccm_matter()) @@ -251,6 +466,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_ccm_matter, kTLSADLen, evp_aead_open, EVP_aead_aes_128_ccm_matter()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_ccm_matter, kTLSADLen, + evp_aead_seal, EVP_aead_aes_128_ccm_matter()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_ccm_matter, kTLSADLen, + evp_aead_open, EVP_aead_aes_128_ccm_matter()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_128_ctr_hmac_sha256, kTLSADLen, evp_aead_seal, EVP_aead_aes_128_ctr_hmac_sha256()) @@ -258,6 +479,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_128_ctr_hmac_sha256, kTLSADLen, evp_aead_open, EVP_aead_aes_128_ctr_hmac_sha256()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_128_ctr_hmac_sha256, kTLSADLen, + evp_aead_seal, EVP_aead_aes_128_ctr_hmac_sha256()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_128_ctr_hmac_sha256, kTLSADLen, + evp_aead_open, EVP_aead_aes_128_ctr_hmac_sha256()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_aes_256_ctr_hmac_sha256, kTLSADLen, evp_aead_seal, EVP_aead_aes_256_ctr_hmac_sha256()) @@ -265,6 +492,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_aes_256_ctr_hmac_sha256, kTLSADLen, evp_aead_open, EVP_aead_aes_256_ctr_hmac_sha256()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_aes_256_ctr_hmac_sha256, kTLSADLen, + evp_aead_seal, EVP_aead_aes_256_ctr_hmac_sha256()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_aes_256_ctr_hmac_sha256, kTLSADLen, + evp_aead_open, EVP_aead_aes_256_ctr_hmac_sha256()) + ->Apply(SetInputLengthv); BENCHMARK_CAPTURE(BM_SpeedAEAD, seal_des_ede3_cbc_sha1, kLegacyADLen, evp_aead_seal, EVP_aead_des_ede3_cbc_sha1_tls()) @@ -272,6 +505,12 @@ BENCHMARK_CAPTURE(BM_SpeedAEAD, open_des_ede3_cbc_sha1, kLegacyADLen, evp_aead_open, EVP_aead_des_ede3_cbc_sha1_tls()) ->Apply(SetInputLength); + BENCHMARK_CAPTURE(BM_SpeedAEADv, seal_des_ede3_cbc_sha1, kLegacyADLen, + evp_aead_seal, EVP_aead_des_ede3_cbc_sha1_tls()) + ->Apply(SetInputLengthv); + BENCHMARK_CAPTURE(BM_SpeedAEADv, open_des_ede3_cbc_sha1, kLegacyADLen, + evp_aead_open, EVP_aead_des_ede3_cbc_sha1_tls()) + ->Apply(SetInputLengthv); } } // namespace