Add support for Chromium's JSON test result format.
Also adds a flag to runner.go to make it more suitable for printing to a pipe.
Change-Id: I26fae21f3e4910028f6b8bfc4821c8c595525504
Reviewed-on: https://boringssl-review.googlesource.com/3490
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go
index 99c66a4..ae7e0e2 100644
--- a/ssl/test/runner/runner.go
+++ b/ssl/test/runner/runner.go
@@ -24,11 +24,13 @@
)
var (
- useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
- useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
- flagDebug *bool = flag.Bool("debug", false, "Hexdump the contents of the connection")
- mallocTest *int64 = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
- mallocTestDebug *bool = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
+ useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
+ useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
+ flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection")
+ mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
+ mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
+ jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
+ pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
)
const (
@@ -1798,7 +1800,7 @@
// False Start without session tickets.
testCases = append(testCases, testCase{
- name: "FalseStart-SessionTicketsDisabled",
+ name: "FalseStart-SessionTicketsDisabled" + suffix,
config: Config{
CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
NextProtos: []string{"foo"},
@@ -2134,7 +2136,7 @@
})
testCases = append(testCases, testCase{
testType: clientTest,
- name: "ServerNameExtensionClient",
+ name: "ServerNameExtensionClientMismatch",
config: Config{
Bugs: ProtocolBugs{
ExpectServerName: "mismatch.com",
@@ -2146,7 +2148,7 @@
})
testCases = append(testCases, testCase{
testType: clientTest,
- name: "ServerNameExtensionClient",
+ name: "ServerNameExtensionClientMissing",
config: Config{
Bugs: ProtocolBugs{
ExpectServerName: "missing.com",
@@ -2576,7 +2578,7 @@
})
testCases = append(testCases, testCase{
protocol: dtls,
- name: "FastRadio-Padding",
+ name: "FastRadio-Padding-DTLS",
config: Config{
Bugs: ProtocolBugs{
RequireFastradioPadding: true,
@@ -2805,27 +2807,43 @@
err error
}
-func statusPrinter(doneChan chan struct{}, statusChan chan statusMsg, total int) {
+func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) {
var started, done, failed, lineLen int
- defer close(doneChan)
+ testOutput := newTestOutput()
for msg := range statusChan {
+ if !*pipe {
+ // Erase the previous status line.
+ fmt.Printf("\x1b[%dD\x1b[K", lineLen)
+ }
+
if msg.started {
started++
} else {
done++
+
+ if msg.err != nil {
+ fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
+ failed++
+ testOutput.addResult(msg.test.name, "FAIL")
+ } else {
+ if *pipe {
+ // Print each test instead of a status line.
+ fmt.Printf("PASSED (%s)\n", msg.test.name)
+ }
+ testOutput.addResult(msg.test.name, "PASS")
+ }
}
- fmt.Printf("\x1b[%dD\x1b[K", lineLen)
-
- if msg.err != nil {
- fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
- failed++
+ if !*pipe {
+ // Print a new status line.
+ line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total)
+ lineLen = len(line)
+ os.Stdout.WriteString(line)
}
- line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total)
- lineLen = len(line)
- os.Stdout.WriteString(line)
}
+
+ doneChan <- testOutput
}
func main() {
@@ -2865,7 +2883,7 @@
statusChan := make(chan statusMsg, numWorkers)
testChan := make(chan *testCase, numWorkers)
- doneChan := make(chan struct{})
+ doneChan := make(chan *testOutput)
go statusPrinter(doneChan, statusChan, len(testCases))
@@ -2883,7 +2901,13 @@
close(testChan)
wg.Wait()
close(statusChan)
- <-doneChan
+ testOutput := <-doneChan
fmt.Printf("\n")
+
+ if *jsonOutput != "" {
+ if err := testOutput.writeTo(*jsonOutput); err != nil {
+ fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+ }
+ }
}
diff --git a/ssl/test/runner/test_output.go b/ssl/test/runner/test_output.go
new file mode 100644
index 0000000..57fbbe6
--- /dev/null
+++ b/ssl/test/runner/test_output.go
@@ -0,0 +1,69 @@
+/* Copyright (c) 2015, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+package main
+
+import (
+ "encoding/json"
+ "os"
+ "time"
+)
+
+// testOutput is a representation of Chromium's JSON test result format. See
+// https://www.chromium.org/developers/the-json-test-results-format
+type testOutput struct {
+ Version int `json:"version"`
+ Interrupted bool `json:"interrupted"`
+ PathDelimiter string `json:"path_delimiter"`
+ SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
+ NumFailuresByType map[string]int `json:"num_failures_by_type"`
+ Tests map[string]testResult `json:"tests"`
+}
+
+type testResult struct {
+ Actual string `json:"actual"`
+ Expected string `json:"expected"`
+}
+
+func newTestOutput() *testOutput {
+ return &testOutput{
+ Version: 3,
+ PathDelimiter: ".",
+ SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
+ NumFailuresByType: make(map[string]int),
+ Tests: make(map[string]testResult),
+ }
+}
+
+func (t *testOutput) addResult(name, result string) {
+ if _, found := t.Tests[name]; found {
+ panic(name)
+ }
+ t.Tests[name] = testResult{Actual: result, Expected: "PASS"}
+ t.NumFailuresByType[result]++
+}
+
+func (t *testOutput) writeTo(name string) error {
+ file, err := os.Create(name)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ out, err := json.MarshalIndent(t, "", " ")
+ if err != nil {
+ return err
+ }
+ _, err = file.Write(out)
+ return err
+}
