runner: enable split handshake tests for TLS 1.3.
Although the new tests are enabled by default, there is a flag to
(continue to) skip them. This is to allow for inter-version
compatibility testing to be performed without a monstrous number of
failures from old versions that don't yet have TLS 1.3 support.
Change-Id: I9f5e201a21f775442859e127c906b5f77ad8755b
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/39388
Commit-Queue: Matt Braithwaite <mab@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/ssl/test/runner/fuzzer_mode.json b/ssl/test/runner/fuzzer_mode.json
index 0a50722..0bdee88 100644
--- a/ssl/test/runner/fuzzer_mode.json
+++ b/ssl/test/runner/fuzzer_mode.json
@@ -44,7 +44,7 @@
"*-EarlyData-RejectUnfinishedWrite-Client-*": "Trial decryption does not work with the NULL cipher.",
"EarlyData-Reject*-Client-*": "Trial decryption does not work with the NULL cipher.",
"CustomExtensions-Server-EarlyDataOffered": "Trial decryption does not work with the NULL cipher.",
- "*-TicketAgeSkew-*-Reject": "Trial decryption does not work with the NULL cipher.",
+ "*-TicketAgeSkew-*-Reject*": "Trial decryption does not work with the NULL cipher.",
"Renegotiate-Client-BadExt*": "Fuzzer mode does not check renegotiation_info.",
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go
index 20866d7..a642bcd 100644
--- a/ssl/test/runner/runner.go
+++ b/ssl/test/runner/runner.go
@@ -50,28 +50,29 @@
)
var (
- useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
- useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
- useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
- flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection")
- mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
- mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
- jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
- pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
- testToRun = flag.String("test", "", "The pattern to filter tests to run, or empty to run all tests")
- numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
- shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
- handshakerPath = flag.String("handshaker-path", "../../../build/ssl/test/handshaker", "The location of the handshaker binary.")
- resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
- fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
- transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
- idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
- deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
- allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.")
- looseErrors = flag.Bool("loose-errors", false, "If true, allow shims to report an untranslated error code.")
- shimConfigFile = flag.String("shim-config", "", "A config file to use to configure the tests for this shim.")
- includeDisabled = flag.Bool("include-disabled", false, "If true, also runs disabled tests.")
- repeatUntilFailure = flag.Bool("repeat-until-failure", false, "If true, the first selected test will be run repeatedly until failure.")
+ useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
+ useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
+ useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
+ flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection")
+ mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
+ mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
+ jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
+ pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
+ testToRun = flag.String("test", "", "The pattern to filter tests to run, or empty to run all tests")
+ numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
+ shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
+ handshakerPath = flag.String("handshaker-path", "../../../build/ssl/test/handshaker", "The location of the handshaker binary.")
+ resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
+ fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
+ transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
+ idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
+ deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
+ allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.")
+ looseErrors = flag.Bool("loose-errors", false, "If true, allow shims to report an untranslated error code.")
+ shimConfigFile = flag.String("shim-config", "", "A config file to use to configure the tests for this shim.")
+ includeDisabled = flag.Bool("include-disabled", false, "If true, also runs disabled tests.")
+ repeatUntilFailure = flag.Bool("repeat-until-failure", false, "If true, the first selected test will be run repeatedly until failure.")
+ tls13SplitHandshakeTests = flag.Bool("tls13-split-handshake-tests", true, "If true, TLS 1.3 tests that use the handshaker will be performed")
)
// ShimConfigurations is used with the “json” package and represents a shim
@@ -1653,12 +1654,21 @@
for _, test := range tests {
if test.protocol != tls ||
test.testType != serverTest ||
- test.config.MaxVersion >= VersionTLS13 ||
- test.config.MaxVersion < VersionTLS10 ||
- (test.resumeConfig != nil && (test.resumeConfig.MaxVersion < VersionTLS10 || test.resumeConfig.MaxVersion >= VersionTLS13)) ||
+ strings.Contains(test.name, "DelegatedCredentials") ||
+ strings.Contains(test.name, "QUICTransportParams") ||
strings.HasPrefix(test.name, "VersionNegotiation-") {
continue
}
+ // TODO(mab): Remove this when it's no longer needed.
+ //
+ // This flag exists to allow TLS 1.3 support to propagate to old
+ // versions, before enabling cross-version compatibility tests.
+ if !*tls13SplitHandshakeTests &&
+ (test.config.MaxVersion >= VersionTLS13 ||
+ test.config.MaxVersion < VersionTLS10 ||
+ (test.resumeConfig != nil && (test.resumeConfig.MaxVersion < VersionTLS10 || test.resumeConfig.MaxVersion >= VersionTLS13))) {
+ continue
+ }
for _, flag := range test.flags {
if flag == "-implicit-handshake" {
