Move JSON test results code into a common module. We can actually use modules now. Change-Id: I0bd8abaf4e3318069f93fa17e89b4804d03944eb Reviewed-on: https://boringssl-review.googlesource.com/c/33205 Commit-Queue: David Benjamin <davidben@google.com> CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org> Reviewed-by: Adam Langley <agl@google.com>
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go index 9d5ba7f..95fd27c 100644 --- a/ssl/test/runner/runner.go +++ b/ssl/test/runner/runner.go
@@ -42,6 +42,8 @@ "sync" "syscall" "time" + + "boringssl.googlesource.com/boringssl/util/testresult" ) var ( @@ -14661,10 +14663,10 @@ err error } -func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) { +func statusPrinter(doneChan chan *testresult.Results, statusChan chan statusMsg, total int) { var started, done, failed, unimplemented, lineLen int - testOutput := newTestOutput() + testOutput := testresult.NewResults() for msg := range statusChan { if !*pipe { // Erase the previous status line. @@ -14687,18 +14689,22 @@ fmt.Printf("UNIMPLEMENTED (%s)\n", msg.test.name) } unimplemented++ - testOutput.addResult(msg.test.name, "UNIMPLEMENTED") + if *allowUnimplemented { + testOutput.AddSkip(msg.test.name) + } else { + testOutput.AddResult(msg.test.name, "SKIP") + } } else { fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err) failed++ - testOutput.addResult(msg.test.name, "FAIL") + testOutput.AddResult(msg.test.name, "FAIL") } } else { if *pipe { // Print each test instead of a status line. fmt.Printf("PASSED (%s)\n", msg.test.name) } - testOutput.addResult(msg.test.name, "PASS") + testOutput.AddResult(msg.test.name, "PASS") } } @@ -14762,7 +14768,7 @@ statusChan := make(chan statusMsg, *numWorkers) testChan := make(chan *testCase, *numWorkers) - doneChan := make(chan *testOutput) + doneChan := make(chan *testresult.Results) if len(*shimConfigFile) != 0 { encoded, err := ioutil.ReadFile(*shimConfigFile) @@ -14835,16 +14841,12 @@ fmt.Printf("\n") if *jsonOutput != "" { - if err := testOutput.writeTo(*jsonOutput); err != nil { + if err := testOutput.WriteToFile(*jsonOutput); err != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", err) } } - if !*allowUnimplemented && testOutput.NumFailuresByType["UNIMPLEMENTED"] > 0 { - os.Exit(1) - } - - if !testOutput.noneFailed { + if !testOutput.HasUnexpectedResults() { os.Exit(1) } }
diff --git a/ssl/test/runner/test_output.go b/ssl/test/runner/test_output.go deleted file mode 100644 index eb54638..0000000 --- a/ssl/test/runner/test_output.go +++ /dev/null
@@ -1,79 +0,0 @@ -/* Copyright (c) 2015, Google Inc. - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ - -package runner - -import ( - "encoding/json" - "os" - "time" -) - -// testOutput is a representation of Chromium's JSON test result format. See -// https://www.chromium.org/developers/the-json-test-results-format -type testOutput struct { - Version int `json:"version"` - Interrupted bool `json:"interrupted"` - PathDelimiter string `json:"path_delimiter"` - SecondsSinceEpoch float64 `json:"seconds_since_epoch"` - NumFailuresByType map[string]int `json:"num_failures_by_type"` - Tests map[string]testResult `json:"tests"` - noneFailed bool -} - -type testResult struct { - Actual string `json:"actual"` - Expected string `json:"expected"` - IsUnexpected bool `json:"is_unexpected"` -} - -func newTestOutput() *testOutput { - return &testOutput{ - Version: 3, - PathDelimiter: ".", - SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), - NumFailuresByType: make(map[string]int), - Tests: make(map[string]testResult), - noneFailed: true, - } -} - -func (t *testOutput) addResult(name, result string) { - if _, found := t.Tests[name]; found { - panic(name) - } - t.Tests[name] = testResult{ - Actual: result, - Expected: "PASS", - IsUnexpected: result != "PASS", - } - t.NumFailuresByType[result]++ - if result != "PASS" && result != "UNIMPLEMENTED" { - t.noneFailed = false - } -} - -func (t *testOutput) writeTo(name string) error { - file, err := os.Create(name) - if err != nil { - return err - } - defer file.Close() - out, err := json.MarshalIndent(t, "", " ") - if err != nil { - return err - } - _, err = file.Write(out) - return err -}
diff --git a/util/all_tests.go b/util/all_tests.go index 8332b1f..55e1921 100644 --- a/util/all_tests.go +++ b/util/all_tests.go
@@ -30,7 +30,8 @@ "strings" "sync" "syscall" - "time" + + "boringssl.googlesource.com/boringssl/util/testresult" ) // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. @@ -68,23 +69,6 @@ Error error } -// testOutput is a representation of Chromium's JSON test result format. See -// https://www.chromium.org/developers/the-json-test-results-format -type testOutput struct { - Version int `json:"version"` - Interrupted bool `json:"interrupted"` - PathDelimiter string `json:"path_delimiter"` - SecondsSinceEpoch float64 `json:"seconds_since_epoch"` - NumFailuresByType map[string]int `json:"num_failures_by_type"` - Tests map[string]testResult `json:"tests"` -} - -type testResult struct { - Actual string `json:"actual"` - Expected string `json:"expected"` - IsUnexpected bool `json:"is_unexpected"` -} - // sdeCPUs contains a list of CPU code that we run all tests under when *useSDE // is true. var sdeCPUs = []string{ @@ -113,46 +97,6 @@ "crypto", // Support for NEON and crypto extensions. } -func newTestOutput() *testOutput { - return &testOutput{ - Version: 3, - PathDelimiter: ".", - SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), - NumFailuresByType: make(map[string]int), - Tests: make(map[string]testResult), - } -} - -func (t *testOutput) addResult(name, result string) { - if _, found := t.Tests[name]; found { - panic(name) - } - expected := "PASS" - if result == "SKIP" { - expected = "SKIP" - } - t.Tests[name] = testResult{ - Actual: result, - Expected: expected, - IsUnexpected: result != expected, - } - t.NumFailuresByType[result]++ -} - -func (t *testOutput) writeTo(name string) error { - file, err := os.Create(name) - if err != nil { - return err - } - defer file.Close() - out, err := json.MarshalIndent(t, "", " ") - if err != nil { - return err - } - _, err = file.Write(out) - return err -} - func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"} if dbAttach { @@ -480,7 +424,7 @@ close(results) }() - testOutput := newTestOutput() + testOutput := testresult.NewResults() var failed, skipped []test for testResult := range results { test := testResult.Test @@ -490,25 +434,25 @@ fmt.Printf("%s\n", test.longName()) fmt.Printf("%s was skipped\n", args[0]) skipped = append(skipped, test) - testOutput.addResult(test.longName(), "SKIP") + testOutput.AddSkip(test.longName()) } else if testResult.Error != nil { fmt.Printf("%s\n", test.longName()) fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error) failed = append(failed, test) - testOutput.addResult(test.longName(), "CRASH") + testOutput.AddResult(test.longName(), "CRASH") } else if !testResult.Passed { fmt.Printf("%s\n", test.longName()) fmt.Printf("%s failed to print PASS on the last line.\n", args[0]) failed = append(failed, test) - testOutput.addResult(test.longName(), "FAIL") + testOutput.AddResult(test.longName(), "FAIL") } else { fmt.Printf("%s\n", test.shortName()) - testOutput.addResult(test.longName(), "PASS") + testOutput.AddResult(test.longName(), "PASS") } } if *jsonOutput != "" { - if err := testOutput.writeTo(*jsonOutput); err != nil { + if err := testOutput.WriteToFile(*jsonOutput); err != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", err) } }
diff --git a/util/testresult/testresult.go b/util/testresult/testresult.go new file mode 100644 index 0000000..5226716 --- /dev/null +++ b/util/testresult/testresult.go
@@ -0,0 +1,95 @@ +/* Copyright (c) 2018, Google Inc. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +// testresult is an implementation of Chromium's JSON test result format. See +// https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md +package testresult + +import ( + "encoding/json" + "os" + "time" +) + +// Results stores the top-level test results. +type Results struct { + Version int `json:"version"` + Interrupted bool `json:"interrupted"` + PathDelimiter string `json:"path_delimiter"` + SecondsSinceEpoch float64 `json:"seconds_since_epoch"` + NumFailuresByType map[string]int `json:"num_failures_by_type"` + Tests map[string]Result `json:"tests"` +} + +func NewResults() *Results { + return &Results{ + Version: 3, + PathDelimiter: ".", + SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), + NumFailuresByType: make(map[string]int), + Tests: make(map[string]Result), + } +} + +func (t *Results) addResult(name, result, expected string) { + if _, found := t.Tests[name]; found { + panic(name) + } + t.Tests[name] = Result{ + Actual: result, + Expected: expected, + IsUnexpected: result != expected, + } + t.NumFailuresByType[result]++ +} + +// AddResult records a test result with the given result string. The test is a +// failure if the result is not "PASS". +func (t *Results) AddResult(name, result string) { + t.addResult(name, result, "PASS") +} + +// AddSkip marks a test as being skipped. It is not considered a failure. +func (t *Results) AddSkip(name string) { + t.addResult(name, "SKIP", "SKIP") +} + +func (t *Results) HasUnexpectedResults() bool { + for _, r := range t.Tests { + if r.IsUnexpected { + return false + } + } + return true +} + +func (t *Results) WriteToFile(name string) error { + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + out, err := json.MarshalIndent(t, "", " ") + if err != nil { + return err + } + _, err = file.Write(out) + return err +} + +type Result struct { + Actual string `json:"actual"` + Expected string `json:"expected"` + IsUnexpected bool `json:"is_unexpected"` +}