diff --git a/.gitignore b/.gitignore index d59a62b7e1..fa397f98d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ build tmp +*.gcov diff --git a/CHANGES b/CHANGES index cce85749d0..fabb87ba43 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,15 @@ +2.5-826 | 2018-08-08 13:09:27 -0700 + + * Add support for code coverage statistics for bro source files after running btest + test suite + + This adds --enable-coverage flag to configure Bro with gcov. + A new directory named /testing/coverage/ contains a new + coverage target. By default a coverage.log is created; running + make html in testing/coverage creates a HTML report. + (Chung Min Kim, Corelight) + 2.5-819 | 2018-08-08 13:03:22 -0500 * Fix cluster layout graphic and doc warnings (Jon Siwek, Corelight) @@ -32,7 +43,7 @@ * Add 'W' connection history indicator for zero windows (Vern Paxson, Corelight) - + * Allow logarithmic 'T'/'C'/'W' connection history repetitions, which also now raise their own events (Vern Paxson, Corelight) diff --git a/VERSION b/VERSION index cd1b9210ec..ca5b6f51d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-819 +2.5-826 diff --git a/configure b/configure index 46aaeb3bbd..90dda2fdd7 100755 --- a/configure +++ b/configure @@ -45,6 +45,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode (like --build-type=Debug) + --enable-coverage compile with code coverage support (implies debugging mode) --enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275 --enable-perftools force use of Google perftools on non-Linux systems (automatically on when perftools is present on Linux) @@ -141,6 +142,8 @@ append_cache_entry INSTALL_BROCTL BOOL true append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING append_cache_entry ENABLE_MOBILE_IPV6 BOOL false append_cache_entry DISABLE_PERFTOOLS BOOL false +append_cache_entry DISABLE_RUBY_BINDINGS BOOL true +append_cache_entry ENABLE_COVERAGE BOOL false # parse arguments while [ $# -ne 0 ]; do @@ -196,6 +199,10 @@ while [ $# -ne 0 ]; do --logdir=*) append_cache_entry BRO_LOG_DIR PATH $optarg ;; + --enable-coverage) + append_cache_entry ENABLE_COVERAGE BOOL true + append_cache_entry ENABLE_DEBUG BOOL true + ;; --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; diff --git a/testing/Makefile b/testing/Makefile index e83ec09396..98c6b239a2 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -8,6 +8,7 @@ brief: make-brief coverage distclean: @rm -f coverage.log $(MAKE) -C btest $@ + $(MAKE) -C coverage $@ make-verbose: @for repo in $(DIRS); do (cd $$repo && make -s ); done @@ -22,4 +23,6 @@ coverage: @echo "Complete test suite code coverage:" @./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd`/../scripts @rm -f brocov.tmp.* + @cd coverage && make coverage +.PHONY: coverage diff --git a/testing/btest/Makefile b/testing/btest/Makefile index c9bcfff5ee..c6f2438ad1 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -21,6 +21,7 @@ coverage: cleanup: @rm -f $(DIAG) @rm -rf $(SCRIPT_COV)* + @find ../../ -name "*.gcda" -exec rm {} \; distclean: cleanup @rm -rf .btest.failed.dat \ diff --git a/testing/coverage/Makefile b/testing/coverage/Makefile new file mode 100644 index 0000000000..7f458a4f9c --- /dev/null +++ b/testing/coverage/Makefile @@ -0,0 +1,12 @@ +coverage: cleanup + @./code_coverage.sh + +cleanup: + @rm -f coverage.log + @find ../../ -name "*.gcov" -exec rm {} \; + +distclean: cleanup + @find ../../ -name "*.gcno" -exec rm {} \; + +html: + @./lcov_html.sh $(COVERAGE_HTML_DIR) diff --git a/testing/coverage/README b/testing/coverage/README new file mode 100644 index 0000000000..d1352640f2 --- /dev/null +++ b/testing/coverage/README @@ -0,0 +1,21 @@ +On a Bro build configured with --enable-coverage, this script produces a code +coverage report after Bro has been invoked. The intended application of this +script is after the btest testsuite has run. This combination (btests first, +coverage computation afterward) happens automatically when running "make" in +the testing directory. This script puts .gcov files (which are included in +.gitignore) alongside the corresponding source files. + +This depends on gcov, which should come with your gcc. If gcov is not +installed, the script will abort with an error message. + +After `make all` in the upper directory, use `make html` as make target in this +directory to output the html files that lcov can create. By default, the html +files will be contained in a directory named "coverage-html" in the base +directory. To set a custom name, use `make html +COVERAGE_HTML_DIR=custom-dir-name`. + +The script code_coverage.sh is triggered by `make coverage` (included in `make` +in /testing), and its goal is to automate code coverage testing. + +The script lcov_html.sh is triggered by `make html`, and its goal is to create +html files from the aforementioned coverage data. diff --git a/testing/coverage/code_coverage.sh b/testing/coverage/code_coverage.sh new file mode 100755 index 0000000000..758b2fa915 --- /dev/null +++ b/testing/coverage/code_coverage.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# On a Bro build configured with --enable-coverage, this script +# produces a code coverage report after Bro has been invoked. The +# intended application of this script is after the btest testsuite has +# run. This combination (btests first, coverage computation afterward) +# happens automatically when running "make" in the testing directory. +# +# This depends on gcov, which should come with your gcc. +# +# AUTOMATES CODE COVERAGE TESTING +# 1. Run test suite +# 2. Check for .gcda files existing. +# 3a. Run gcov (-p to preserve path) +# 3b. Prune .gcov files for objects outside of the Bro tree +# 4a. Analyze .gcov files generated and create summary file +# 4b. Send .gcov files to appropriate path +# +CURR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Location of script +BASE="$( cd "$CURR" && cd ../../ && pwd )" +TMP="${CURR}/tmp.$$" +mkdir -p $TMP + +# DEFINE CLEANUP PROCESS +function finish { + rm -rf $TMP +} +trap finish EXIT + +# DEFINE CRUCIAL FUNCTIONS FOR COVERAGE CHECKING +function check_file_coverage { + GCOVDIR="$1" + + for i in $GCOVDIR/*.gcov; do + # Effective # of lines: starts with a number (# of runs in line) or ##### (line never run) + TOTAL=$(cut -d: -f 1 "$i" | sed 's/ //g' | grep -v "^[[:alpha:]]" | grep -v "-" | wc -l) + + # Count number of lines never run + UNRUN=$(grep "#####" "$i" | wc -l) + + # Lines in code are either run or unrun + RUN=$(($TOTAL - $UNRUN)) + + # Avoid division-by-zero problems: + PERCENTAGE=0.000 + [ $RUN -gt 0 ] && PERCENTAGE=$(bc <<< "scale=3; 100*$RUN/$TOTAL") + + # Find correlation between % of lines run vs. "Runs" + echo -e "$PERCENTAGE\t$RUN\t$TOTAL\t$(grep "0:Runs" "$i" | sed 's/.*://')\t$i" + done +} + +function check_group_coverage { + DATA="$1" # FILE CONTAINING COVERAGE DATA + SRC_FOLDER="$2" # WHERE BRO WAS COMPILED + OUTPUT="$3" + + # Prints all the relevant directories + DIRS=$(for i in $(cut -f 5 "$DATA"); do basename "$i" | sed 's/#[^#]*$//'; done \ + | sort | uniq | sed 's/^.*'"${SRC_FOLDER}"'//' | grep "^#s\+" ) + # "Generalize" folders unless it's from analyzers + DIRS=$(for i in $DIRS; do + if !(echo "$i" | grep "src#analyzer"); then + echo "$i" | cut -d "#" -f 1,2,3 + fi + done | sort | uniq ) + + for i in $DIRS; do + # For elements in #src, we only care about the files direclty in the directory. + if [[ "$i" = "#src" ]]; then + RUN=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 2) | tr " " "+" | bc) + TOTAL=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 3) | tr " " "+" | bc) + else + RUN=$(echo $(grep "$i" $DATA | cut -f 2) | tr " " "+" | bc) + TOTAL=$(echo $(grep "$i" $DATA | cut -f 3) | tr " " "+" | bc) + fi + + PERCENTAGE=$( echo "scale=3;100*$RUN/$TOTAL" | bc | tr "\n" " " ) + printf "%-50s\t%12s\t%6s %%\n" "$i" "$RUN/$TOTAL" $PERCENTAGE \ + | sed 's|#|/|g' >>$OUTPUT + done +} + +# 1. Run test suite +# SHOULD HAVE ALREADY BEEN RUN BEFORE THIS SCRIPT (BASED ON MAKEFILE TARGETS) + +# 2. Check for .gcno and .gcda file presence +echo -n "Checking for coverage files... " +for pat in gcda gcno; do + if [ -z "$(find "$BASE" -name "*.$pat" 2>/dev/null)" ]; then + echo "no .$pat files, nothing to do" + exit 0 + fi +done +echo "ok" + +# 3a. Run gcov (-p to preserve path) and move into tmp directory +# ... if system does not have gcov installed, exit with message. +echo -n "Creating coverage files... " +if which gcov > /dev/null 2>&1; then + ( cd "$TMP" && find "$BASE" -name "*.o" -exec gcov -p {} > /dev/null 2>&1 \; ) + NUM_GCOVS=$(find "$TMP" -name *.gcov | wc -l) + if [ $NUM_GCOVS -eq 0 ]; then + echo "no gcov files produced, aborting" + exit 1 + fi + + # Account for '^' that occurs in macOS due to LLVM + # This character seems to be equivalent to ".." (up 1 dir) + for file in $(ls $TMP/*.gcov | grep '\^'); do + mv $file "$(sed 's/#[^#]*#\^//g' <<< "$file")" + done + + echo "ok, $NUM_GCOVS coverage files" +else + echo "gcov is not installed on system, aborting" + exit 1 +fi + +# 3b. Prune gcov files that fall outside of the Bro tree: +# Look for files containing gcov's slash substitution character "#" +# and remove any that don't contain the Bro path root. +echo -n "Pruning out-of-tree coverage files... " +PREFIX=$(echo "$BASE" | sed 's|/|#|g') +for i in "$TMP"/*#*.gcov; do + if ! [[ "$i" = *$PREFIX* ]]; then + rm -f $i + fi +done +NUM_GCOVS=$(ls "$TMP"/*.gcov | wc -l) +echo "ok, $NUM_GCOVS coverage files remain" + +# 4a. Analyze .gcov files generated and create summary file +echo -n "Creating summary file... " +DATA="${TMP}/data.txt" +SUMMARY="$CURR/coverage.log" +check_file_coverage "$TMP" > "$DATA" +check_group_coverage "$DATA" ${BASE##*/} $SUMMARY +echo "ok" + +# 4b. Send .gcov files to appropriate path +echo -n "Sending coverage files to respective directories... " +for i in "$TMP"/*#*.gcov; do + mv $i $(echo $(basename $i) | sed 's/#/\//g') +done +echo "ok" diff --git a/testing/coverage/lcov_html.sh b/testing/coverage/lcov_html.sh new file mode 100755 index 0000000000..c729b2145c --- /dev/null +++ b/testing/coverage/lcov_html.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# On a Bro build configured with --enable-coverage, this script +# produces a code coverage report in HTML format after Bro has been invoked. The +# intended application of this script is after the btest testsuite has run. + +# This depends on lcov to run. + +function die { + echo "$@" + exit 1 +} +function finish { + rm -rf "$TMP" +} +function verify_run { + if bash -c "$1" > /dev/null 2>&1; then + echo ${2:-"ok"} + else + die ${3:-"error, abort"} + fi +} +trap finish EXIT + +TMP=".tmp.$$" +COVERAGE_FILE="./$TMP/coverage.info" +COVERAGE_HTML_DIR="${1:-"coverage-html"}" +REMOVE_TARGETS="*.yy *.ll *.y *.l */bro.dir/* *.bif" + +# 1. Move to base dir, create tmp dir +cd ../../; +mkdir "$TMP" + +# 2. Check for .gcno and .gcda file presence +echo -n "Checking for coverage files... " +for pat in gcda gcno; do + if [ -z "$(find . -name "*.$pat" 2>/dev/null)" ]; then + echo "no .$pat files, nothing to do" + exit 0 + fi +done +echo "ok" + +# 3. If lcov does not exist, abort process. +echo -n "Checking for lcov... " +verify_run "which lcov" \ + "lcov installed on system, continue" \ + "lcov not installed, abort" + +# 4. Create a "tracefile" through lcov, which is necessary to create html files later on. +echo -n "Creating tracefile for html generation... " +verify_run "lcov --no-external --capture --directory . --output-file $COVERAGE_FILE" + +for TARGET in $REMOVE_TARGETS; do + echo -n "Getting rid of $TARGET files from tracefile... " + verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE" +done + +# 5. Create HTML files. +echo -n "Creating HTML files... " +verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE"