tool: add -json flag to |speed| Add a flag to speed.cc to generate machine-readable benchmark results. Change-Id: I24a4324c5195b15494dc6d9471aa91c27b9f881d Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/37865 Reviewed-by: Adam Langley <alangley@gmail.com> Reviewed-by: Zola Bridges <zbrid@google.com> Reviewed-by: David Benjamin <davidben@google.com> Commit-Queue: David Benjamin <davidben@google.com>
diff --git a/tool/speed.cc b/tool/speed.cc index 160d90f..68073a9 100644 --- a/tool/speed.cc +++ b/tool/speed.cc
@@ -13,9 +13,9 @@ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include <algorithm> -#include <string> #include <functional> #include <memory> +#include <string> #include <vector> #include <assert.h> @@ -54,6 +54,8 @@ #include "../third_party/sike/sike.h" +// g_print_json is true if printed output is JSON formatted. +static bool g_print_json = false; // TimeResults represents the results of benchmarking a function. struct TimeResults { @@ -62,20 +64,54 @@ // us is the number of microseconds that elapsed in the time period. unsigned us; - void Print(const std::string &description) { - printf("Did %u %s operations in %uus (%.1f ops/sec)\n", num_calls, - description.c_str(), us, - (static_cast<double>(num_calls) / us) * 1000000); + void Print(const std::string &description) const { + if (g_print_json) { + PrintJSON(description); + } else { + printf("Did %u %s operations in %uus (%.1f ops/sec)\n", num_calls, + description.c_str(), us, + (static_cast<double>(num_calls) / us) * 1000000); + } } - void PrintWithBytes(const std::string &description, size_t bytes_per_call) { - printf("Did %u %s operations in %uus (%.1f ops/sec): %.1f MB/s\n", - num_calls, description.c_str(), us, - (static_cast<double>(num_calls) / us) * 1000000, - static_cast<double>(bytes_per_call * num_calls) / us); + void PrintWithBytes(const std::string &description, + size_t bytes_per_call) const { + if (g_print_json) { + PrintJSON(description, bytes_per_call); + } else { + printf("Did %u %s operations in %uus (%.1f ops/sec): %.1f MB/s\n", + num_calls, description.c_str(), us, + (static_cast<double>(num_calls) / us) * 1000000, + static_cast<double>(bytes_per_call * num_calls) / us); + } } + + private: + void PrintJSON(const std::string &description, + size_t bytes_per_call = 0) const { + if (first_json_printed) { + puts(","); + } + + printf("{\"description\": \"%s\", \"numCalls\": %u, \"microseconds\": %u", + description.c_str(), num_calls, us); + + if (bytes_per_call > 0) { + printf(", \"bytesPerCall\": %zu", bytes_per_call); + } + + printf("}"); + first_json_printed = true; + } + + // first_json_printed is true if |g_print_json| is true and the first item in + // the JSON results has been printed already. This is used to handle the + // commas between each item in the result list. + static bool first_json_printed; }; +bool TimeResults::first_json_printed = false; + #if defined(OPENSSL_WINDOWS) static uint64_t time_now() { return GetTickCount64() * 1000; } #elif defined(OPENSSL_APPLE) @@ -274,24 +310,29 @@ } std::sort(durations.begin(), durations.end()); - printf("Did %u RSA %d key-gen operations in %uus (%.1f ops/sec)\n", - num_calls, size, us, - (static_cast<double>(num_calls) / us) * 1000000); + const std::string description = + std::string("RSA ") + std::to_string(size) + std::string(" key-gen"); + const TimeResults results = {num_calls, us}; + results.Print(description); const size_t n = durations.size(); assert(n > 0); - // |min| and |max| must be stored in temporary variables to avoid an MSVC - // bug on x86. There, size_t is a typedef for unsigned, but MSVC's printf - // warning tries to retain the distinction and suggest %zu for size_t - // instead of %u. It gets confused if std::vector<unsigned> and - // std::vector<size_t> are both instantiated. Being typedefs, the two - // instantiations are identical, which somehow breaks the size_t vs unsigned - // metadata. - unsigned min = durations[0]; - unsigned median = n & 1 ? durations[n / 2] - : (durations[n / 2 - 1] + durations[n / 2]) / 2; - unsigned max = durations[n - 1]; - printf(" min: %uus, median: %uus, max: %uus\n", min, median, max); + // Distribution information is useful, but doesn't fit into the standard + // format used by |g_print_json|. + if (!g_print_json) { + // |min| and |max| must be stored in temporary variables to avoid an MSVC + // bug on x86. There, size_t is a typedef for unsigned, but MSVC's printf + // warning tries to retain the distinction and suggest %zu for size_t + // instead of %u. It gets confused if std::vector<unsigned> and + // std::vector<size_t> are both instantiated. Being typedefs, the two + // instantiations are identical, which somehow breaks the size_t vs + // unsigned metadata. + unsigned min = durations[0]; + unsigned median = n & 1 ? durations[n / 2] + : (durations[n / 2 - 1] + durations[n / 2]) / 2; + unsigned max = durations[n - 1]; + printf(" min: %uus, median: %uus, max: %uus\n", min, median, max); + } } return true; @@ -979,6 +1020,16 @@ "16,256,1350,8192,16384)", }, { + "-json", + kBooleanArgument, + "If this flag is set, speed will print the output of each benchmark in " + "JSON format as follows: \"{\"description\": " + "\"descriptionOfOperation\", \"numCalls\": 1234, " + "\"timeInMicroseconds\": 1234567, \"bytesPerCall\": 1234}\". When " + "there is no information about the bytes per call for an operation, " + "the JSON field for bytesPerCall will be omitted.", + }, + { "", kOptionalArgument, "", @@ -997,6 +1048,10 @@ selected = args_map["-filter"]; } + if (args_map.count("-json") != 0) { + g_print_json = true; + } + if (args_map.count("-timeout") != 0) { g_timeout_seconds = atoi(args_map["-timeout"].c_str()); } @@ -1036,6 +1091,9 @@ // knowledge in them and construct a couple of the AD bytes internally. static const size_t kLegacyADLen = kTLSADLen - 2; + if (g_print_json) { + puts("["); + } if (!SpeedRSA(selected) || !SpeedAEAD(EVP_aead_aes_128_gcm(), "AES-128-GCM", kTLSADLen, selected) || !SpeedAEAD(EVP_aead_aes_256_gcm(), "AES-256-GCM", kTLSADLen, selected) || @@ -1077,6 +1135,9 @@ !SpeedHRSS(selected)) { return false; } + if (g_print_json) { + puts("\n]"); + } return true; }