runner: enable split handshake tests for TLS 1.3. Although the new tests are enabled by default, there is a flag to (continue to) skip them. This is to allow for inter-version compatibility testing to be performed without a monstrous number of failures from old versions that don't yet have TLS 1.3 support. Change-Id: I9f5e201a21f775442859e127c906b5f77ad8755b Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/39388 Commit-Queue: Matt Braithwaite <mab@google.com> Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/ssl/test/runner/fuzzer_mode.json b/ssl/test/runner/fuzzer_mode.json index 0a50722..0bdee88 100644 --- a/ssl/test/runner/fuzzer_mode.json +++ b/ssl/test/runner/fuzzer_mode.json
@@ -44,7 +44,7 @@ "*-EarlyData-RejectUnfinishedWrite-Client-*": "Trial decryption does not work with the NULL cipher.", "EarlyData-Reject*-Client-*": "Trial decryption does not work with the NULL cipher.", "CustomExtensions-Server-EarlyDataOffered": "Trial decryption does not work with the NULL cipher.", - "*-TicketAgeSkew-*-Reject": "Trial decryption does not work with the NULL cipher.", + "*-TicketAgeSkew-*-Reject*": "Trial decryption does not work with the NULL cipher.", "Renegotiate-Client-BadExt*": "Fuzzer mode does not check renegotiation_info.",
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go index 20866d7..a642bcd 100644 --- a/ssl/test/runner/runner.go +++ b/ssl/test/runner/runner.go
@@ -50,28 +50,29 @@ ) var ( - useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") - useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") - useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb") - flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection") - mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") - mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") - jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") - pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.") - testToRun = flag.String("test", "", "The pattern to filter tests to run, or empty to run all tests") - numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.") - shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.") - handshakerPath = flag.String("handshaker-path", "../../../build/ssl/test/handshaker", "The location of the handshaker binary.") - resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.") - fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.") - transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.") - idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.") - deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.") - allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.") - looseErrors = flag.Bool("loose-errors", false, "If true, allow shims to report an untranslated error code.") - shimConfigFile = flag.String("shim-config", "", "A config file to use to configure the tests for this shim.") - includeDisabled = flag.Bool("include-disabled", false, "If true, also runs disabled tests.") - repeatUntilFailure = flag.Bool("repeat-until-failure", false, "If true, the first selected test will be run repeatedly until failure.") + useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") + useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") + useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb") + flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection") + mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") + mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") + jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") + pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.") + testToRun = flag.String("test", "", "The pattern to filter tests to run, or empty to run all tests") + numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.") + shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.") + handshakerPath = flag.String("handshaker-path", "../../../build/ssl/test/handshaker", "The location of the handshaker binary.") + resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.") + fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.") + transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.") + idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.") + deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.") + allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.") + looseErrors = flag.Bool("loose-errors", false, "If true, allow shims to report an untranslated error code.") + shimConfigFile = flag.String("shim-config", "", "A config file to use to configure the tests for this shim.") + includeDisabled = flag.Bool("include-disabled", false, "If true, also runs disabled tests.") + repeatUntilFailure = flag.Bool("repeat-until-failure", false, "If true, the first selected test will be run repeatedly until failure.") + tls13SplitHandshakeTests = flag.Bool("tls13-split-handshake-tests", true, "If true, TLS 1.3 tests that use the handshaker will be performed") ) // ShimConfigurations is used with the “json” package and represents a shim @@ -1653,12 +1654,21 @@ for _, test := range tests { if test.protocol != tls || test.testType != serverTest || - test.config.MaxVersion >= VersionTLS13 || - test.config.MaxVersion < VersionTLS10 || - (test.resumeConfig != nil && (test.resumeConfig.MaxVersion < VersionTLS10 || test.resumeConfig.MaxVersion >= VersionTLS13)) || + strings.Contains(test.name, "DelegatedCredentials") || + strings.Contains(test.name, "QUICTransportParams") || strings.HasPrefix(test.name, "VersionNegotiation-") { continue } + // TODO(mab): Remove this when it's no longer needed. + // + // This flag exists to allow TLS 1.3 support to propagate to old + // versions, before enabling cross-version compatibility tests. + if !*tls13SplitHandshakeTests && + (test.config.MaxVersion >= VersionTLS13 || + test.config.MaxVersion < VersionTLS10 || + (test.resumeConfig != nil && (test.resumeConfig.MaxVersion < VersionTLS10 || test.resumeConfig.MaxVersion >= VersionTLS13))) { + continue + } for _, flag := range test.flags { if flag == "-implicit-handshake" {