Adding scripts to generate line coverage.
Uses LCOV for C(++) line coverage and Valgrind's Callgrind tool to
generate assembly-level line coverage for the generated assembly
code.
BUG=590332
Change-Id: Ic70300a272c38f4fa6dd615747db568aa0853584
Reviewed-on: https://boringssl-review.googlesource.com/7251
Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/util/all_tests.go b/util/all_tests.go
index 38899bb..d361651 100644
--- a/util/all_tests.go
+++ b/util/all_tests.go
@@ -33,6 +33,7 @@
var (
useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
+ useCallgrind = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.")
useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.")
numWorkers = flag.Int("num-workers", 1, "Runs the given number of workers when testing.")
@@ -113,6 +114,14 @@
return exec.Command("valgrind", valgrindArgs...)
}
+func callgrindOf(path string, args ...string) *exec.Cmd {
+ valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"}
+ valgrindArgs = append(valgrindArgs, path)
+ valgrindArgs = append(valgrindArgs, args...)
+
+ return exec.Command("valgrind", valgrindArgs...)
+}
+
func gdbOf(path string, args ...string) *exec.Cmd {
xtermArgs := []string{"-e", "gdb", "--args"}
xtermArgs = append(xtermArgs, path)
@@ -135,6 +144,8 @@
var cmd *exec.Cmd
if *useValgrind {
cmd = valgrindOf(false, prog, args...)
+ } else if *useCallgrind {
+ cmd = callgrindOf(prog, args...)
} else if *useGDB {
cmd = gdbOf(prog, args...)
} else {
diff --git a/util/generate-asm-lcov.py b/util/generate-asm-lcov.py
new file mode 100755
index 0000000..7b66070
--- /dev/null
+++ b/util/generate-asm-lcov.py
@@ -0,0 +1,149 @@
+#!/usr/bin/python
+# Copyright (c) 2016, Google Inc.
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+import os
+import os.path
+import subprocess
+import sys
+
+# The LCOV output format for each source file is:
+#
+# SF:<filename>
+# DA:<line>,<execution count>
+# ...
+# end_of_record
+#
+# The <execution count> can either be 0 for an unexecuted instruction or a
+# value representing the number of executions. The DA line should be omitted
+# for lines not representing an instruction.
+
+SECTION_SEPERATOR = '-' * 80
+
+def is_asm(l):
+ """Returns whether a line should be considered to be an instruction."""
+ l = l.strip()
+ # Empty lines
+ if l == '':
+ return False
+ # Comments
+ if l.startswith('#'):
+ return False
+ # Assembly Macros
+ if l.startswith('.'):
+ return False
+ # Label
+ if l.endswith(':'):
+ return False
+ return True
+
+def merge(callgrind_files, srcs):
+ """Calls callgrind_annotate over the set of callgrind output
+ |callgrind_files| using the sources |srcs| and merges the results
+ together."""
+ out = ''
+ for file in callgrind_files:
+ data = subprocess.check_output(['callgrind_annotate', file] + srcs)
+ out += '%s\n%s\n' % (data, SECTION_SEPERATOR)
+ return out
+
+def parse(filename, data, current):
+ """Parses an annotated execution flow |data| from callgrind_annotate for
+ source |filename| and updates the current execution counts from |current|."""
+ with open(filename) as f:
+ source = f.read().split('\n')
+
+ out = current
+ if out == None:
+ out = [0 if is_asm(l) else None for l in source]
+
+ # Lines are of the following formats:
+ # -- line: Indicates that analysis continues from a different place.
+ # => : Indicates a call/jump in the control flow.
+ # <Count> <Code>: Indicates that the line has been executed that many times.
+ line = None
+ for l in data:
+ l = l.strip() + ' '
+ if l.startswith('-- line'):
+ line = int(l.split(' ')[2]) - 1
+ elif line != None and '=>' not in l:
+ count = l.split(' ')[0].replace(',', '').replace('.', '0')
+ instruction = l.split(' ', 1)[1].strip()
+ if count != '0' or is_asm(instruction):
+ if out[line] == None:
+ out[line] = 0
+ out[line] += int(count)
+ line += 1
+
+ return out
+
+
+def generate(data):
+ """Parses the merged callgrind_annotate output |data| and generates execution
+ counts for all annotated files."""
+ out = {}
+ data = [p.strip() for p in data.split(SECTION_SEPERATOR)]
+
+
+ # Most sections are ignored, but a section with:
+ # User-annotated source: <file>
+ # precedes a listing of execution count for that <file>.
+ for i in range(len(data)):
+ if 'User-annotated source' in data[i] and i < len(data) - 1:
+ filename = data[i].split(':', 1)[1].strip()
+ res = data[i + 1]
+ if filename not in out:
+ out[filename] = None
+ if 'No information' in res:
+ res = []
+ else:
+ res = res.split('\n')
+ out[filename] = parse(filename, res, out[filename])
+ return out
+
+def output(data):
+ """Takes a dictionary |data| of filenames and execution counts and generates
+ a LCOV coverage output."""
+ out = ''
+ for filename, counts in data.iteritems():
+ out += 'SF:%s\n' % (os.path.abspath(filename))
+ for line, count in enumerate(counts):
+ if count != None:
+ out += 'DA:%d,%s\n' % (line + 1, count)
+ out += 'end_of_record\n'
+ return out
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ print '%s <Callgrind Folder> <Build Folder>' % (__file__)
+ sys.exit()
+
+ cg_folder = sys.argv[1]
+ build_folder = sys.argv[2]
+
+ cg_files = []
+ for (cwd, _, files) in os.walk(cg_folder):
+ for f in files:
+ if f.startswith('callgrind.out'):
+ cg_files.append(os.path.abspath(os.path.join(cwd, f)))
+
+ srcs = []
+ for (cwd, _, files) in os.walk(build_folder):
+ for f in files:
+ fn = os.path.join(cwd, f)
+ if fn.endswith('.S'):
+ srcs.append(fn)
+
+ annotated = merge(cg_files, srcs)
+ lcov = generate(annotated)
+ print output(lcov)
diff --git a/util/generate-coverage.sh b/util/generate-coverage.sh
new file mode 100755
index 0000000..2fbe6b8
--- /dev/null
+++ b/util/generate-coverage.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+# Copyright (c) 2016, Google Inc.
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+set -xe
+
+SRC=$PWD
+
+BUILD=$(mktemp -d '/tmp/boringssl.XXXXXX')
+BUILD_SRC=$(mktemp -d '/tmp/boringssl-src.XXXXXX')
+LCOV=$(mktemp -d '/tmp/boringssl-lcov.XXXXXX')
+
+if [ -n "$1" ]; then
+ LCOV=$(readlink -f "$1")
+ mkdir -p "$LCOV"
+fi
+
+cd "$BUILD"
+cmake "$SRC" -GNinja -DCMAKE_C_FLAGS='-fprofile-arcs -ftest-coverage' \
+ -DCMAKE_CXX_FLAGS='-fprofile-arcs -ftest-coverage' -DCMAKE_ASM_FLAGS='-Wa,-g'
+ninja
+
+cp -r "$SRC/crypto" "$SRC/decrepit" "$SRC/include" "$SRC/ssl" "$SRC/tool" \
+ "$BUILD_SRC"
+cp -r "$BUILD"/* "$BUILD_SRC"
+mkdir "$BUILD/callgrind/"
+
+cd "$SRC"
+go run "$SRC/util/all_tests.go" -build-dir "$BUILD" -callgrind -num-workers 16
+util/generate-asm-lcov.py "$BUILD/callgrind" "$BUILD" > "$BUILD/asm.info"
+
+go run "util/all_tests.go" -build-dir "$BUILD"
+
+cd "$SRC/ssl/test/runner"
+go test -shim-path "$BUILD/ssl/test/bssl_shim" -num-workers 1
+
+cd "$LCOV"
+lcov -c -d "$BUILD" -b "$BUILD" -o "$BUILD/lcov.info"
+lcov -r "$BUILD/lcov.info" "*_test.c" -o "$BUILD/lcov-1.info"
+lcov -r "$BUILD/lcov-1.info" "*_test.cc" -o "$BUILD/lcov-2.info"
+cat "$BUILD/lcov-2.info" "$BUILD/asm.info" > "$BUILD/final.info"
+sed -i "s;$BUILD;$BUILD_SRC;g" "$BUILD/final.info"
+sed -i "s;$SRC;$BUILD_SRC;g" "$BUILD/final.info"
+genhtml -p "$BUILD_SRC" "$BUILD/final.info"
+
+rm -rf "$BUILD"
+rm -rf "$BUILD_SRC"
+
+xdg-open index.html
