From 4cdf1e39bbee3dd50a9d4b1a38da4ee78ad3bc22 Mon Sep 17 00:00:00 2001 From: Chung Min Kim Date: Fri, 22 Jun 2018 14:27:46 -0700 Subject: [PATCH] Add code coverage for bro source files after btest test suite Adds --enable-coverage flag to configure Bro with gcov. A new directory named /testing/bro-code-coverage/ contains a new coverage target that as part of `make coverage` in /testing/. This coverage option creates coverage.log of all important directories in /src/ and places all generated .gcov files alongside the corresponding source file. --- configure | 7 ++ testing/Makefile | 2 + testing/bro-core-coverage/Makefile | 9 ++ testing/bro-core-coverage/code_coverage.sh | 134 +++++++++++++++++++++ testing/btest/Makefile | 1 + 5 files changed, 153 insertions(+) create mode 100644 testing/bro-core-coverage/Makefile create mode 100755 testing/bro-core-coverage/code_coverage.sh diff --git a/configure b/configure index fdbca263c6..56d10e393b 100755 --- a/configure +++ b/configure @@ -45,6 +45,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode (like --build-type=Debug) + --enable-coverage compile with code coverage support (implies debugging mode) --enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275 --enable-perftools force use of Google perftools on non-Linux systems (automatically on when perftools is present on Linux) @@ -142,6 +143,8 @@ append_cache_entry INSTALL_BROCTL BOOL true append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING append_cache_entry ENABLE_MOBILE_IPV6 BOOL false append_cache_entry DISABLE_PERFTOOLS BOOL false +append_cache_entry DISABLE_RUBY_BINDINGS BOOL true +append_cache_entry ENABLE_COVERAGE BOOL false # parse arguments while [ $# -ne 0 ]; do @@ -197,6 +200,10 @@ while [ $# -ne 0 ]; do --logdir=*) append_cache_entry BRO_LOG_DIR PATH $optarg ;; + --enable-coverage) + append_cache_entry ENABLE_COVERAGE BOOL true + append_cache_entry ENABLE_DEBUG BOOL true + ;; --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; diff --git a/testing/Makefile b/testing/Makefile index e83ec09396..e34aaca939 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -8,6 +8,7 @@ brief: make-brief coverage distclean: @rm -f coverage.log $(MAKE) -C btest $@ + $(MAKE) -C bro-core-coverage $@ make-verbose: @for repo in $(DIRS); do (cd $$repo && make -s ); done @@ -22,4 +23,5 @@ coverage: @echo "Complete test suite code coverage:" @./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd`/../scripts @rm -f brocov.tmp.* + @cd bro-core-coverage && make coverage diff --git a/testing/bro-core-coverage/Makefile b/testing/bro-core-coverage/Makefile new file mode 100644 index 0000000000..e7db4894fd --- /dev/null +++ b/testing/bro-core-coverage/Makefile @@ -0,0 +1,9 @@ +coverage: cleanup + @./code_coverage.sh + +cleanup: + @rm -f coverage.log + @find ../../ -name "*.gcov" -exec rm {} \; + +distclean: cleanup + @find ../../ -name "*.gcno" -exec rm {} \; diff --git a/testing/bro-core-coverage/code_coverage.sh b/testing/bro-core-coverage/code_coverage.sh new file mode 100755 index 0000000000..b1eb8c599a --- /dev/null +++ b/testing/bro-core-coverage/code_coverage.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# +# On a Bro build configured with --enable-coverage, this script +# produces a code coverage report after Bro has been invoked. The +# intended application of this script is after the btest testsuite has +# run. This combination (btests first, coverage computation afterward) +# happens automatically when running "make" in the testing directory. +# +# This depends on gcov, which should come with your gcc. +# +# AUTOMATES CODE COVERAGE TESTING +# 1. Run test suite +# 2. Check for .gcda files existing. +# 3a. Run gcov (-p to preserve path) +# 3b. Prune .gcov files for objects outside of the Bro tree +# 4a. Analyze .gcov files generated and create summary file +# 4b. Send .gcov files to appropriate path +# +CURR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Location of script +BASE="$(realpath "${CURR}/../../")" +TMP="${CURR}/tmp.$$" +mkdir -p $TMP + +# DEFINE CLEANUP PROCESS +function finish { + find "$BASE" -name "*.gcda" -exec rm {} \; > /dev/null 2>&1 + rm -rf $TMP +} +trap finish EXIT + +# DEFINE CRUCIAL FUNCTIONS FOR COVERAGE CHECKING +function check_file_coverage { + GCOVDIR="$1" + + for i in $GCOVDIR/*.gcov; do + # Effective # of lines: starts with a number (# of runs in line) or ##### (line never run) + TOTAL=$(cut -d: -f 1 "$i" | sed 's/ //g' | grep -v "^[[:alpha:]]" | grep -v "-" | wc -l) + + # Count number of lines never run + UNRUN=$(grep "#####" "$i" | wc -l) + + # Lines in code are either run or unrun + RUN=$(($TOTAL - $UNRUN)) + + # Avoid division-by-zero problems: + PERCENTAGE=0.000 + [ $RUN -gt 0 ] && PERCENTAGE=$(bc <<< "scale=3; 100*$RUN/$TOTAL") + + # Find correlation between % of lines run vs. "Runs" + echo -e "$PERCENTAGE\t$RUN\t$TOTAL\t$(grep "0:Runs" "$i" | sed 's/.*://')\t$i" + done +} + +function check_group_coverage { + DATA="$1" # FILE CONTAINING COVERAGE DATA + SRC_FOLDER="$2" # WHERE BRO WAS COMPILED + OUTPUT="$3" + + # Prints all the relevant directories + DIRS=$(for i in $(cut -f 5 "$DATA"); do basename "$i" | sed 's/#[^#]*$//'; done \ + | sort | uniq | sed 's/^.*'"${SRC_FOLDER}"'//' | grep "^#s\+" ) + # "Generalize" folders unless it's from analyzers + DIRS=$(for i in $DIRS; do + if !(echo "$i" | grep "src#analyzer"); then + echo "$i" | cut -d "#" -f 1,2,3 + fi + done | sort | uniq ) + + for i in $DIRS; do + # For elements in #src, we only care about the files direclty in the directory. + if [[ "$i" = "#src" ]]; then + RUN=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 2) | tr " " "+" | bc) + TOTAL=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 3) | tr " " "+" | bc) + else + RUN=$(echo $(grep "$i" $DATA | cut -f 2) | tr " " "+" | bc) + TOTAL=$(echo $(grep "$i" $DATA | cut -f 3) | tr " " "+" | bc) + fi + + PERCENTAGE=$( echo "scale=3;100*$RUN/$TOTAL" | bc | tr "\n" " " ) + printf "%-50s\t%12s\t%6s %%\n" "$i" "$RUN/$TOTAL" $PERCENTAGE \ + | sed 's|#|/|g' >>$OUTPUT + done +} + +# 1. Run test suite +# SHOULD HAVE ALREADY BEEN RUN BEFORE THIS SCRIPT (BASED ON MAKEFILE TARGETS) + +# 2. Check for .gcno and .gcda file presence +echo -n "Checking for coverage files... " +for pat in gcda gcno; do + if [ -z "$(find "$BASE" -name "*.$pat" 2>/dev/null)" ]; then + echo "no .$pat files, nothing to do" + exit 0 + fi +done +echo "ok" + +# 3a. Run gcov (-p to preserve path) and move into tmp directory +echo -n "Creating coverage files... " +( cd "$TMP" && find "$BASE" -name "*.o" -exec gcov -p {} > /dev/null 2>&1 \; ) +NUM_GCOVS=$(ls "$TMP"/*.gcov | wc -l) +if [ $NUM_GCOVS -eq 0 ]; then + echo "no gcov files produced, aborting" + exit 1 +fi +echo "ok, $NUM_GCOVS coverage files" + +# 3b. Prune gcov files that fall outside of the Bro tree: +# Look for files containing gcov's slash substitution character "#" +# and remove any that don't contain the Bro path root. +echo -n "Pruning out-of-tree coverage files... " +PREFIX=$(echo "$BASE" | sed 's|/|#|g') +for i in "$TMP"/*#*.gcov; do + if ! [[ "$i" = *$PREFIX* ]]; then + rm -f $i + fi +done +NUM_GCOVS=$(ls "$TMP"/*.gcov | wc -l) +echo "ok, $NUM_GCOVS coverage files remain" + +# 4a. Analyze .gcov files generated and create summary file +echo -n "Creating summary file... " +DATA="${TMP}/data.txt" +SUMMARY="$CURR/coverage.log" +check_file_coverage "$TMP" > "$DATA" +check_group_coverage "$DATA" ${BASE##*/} $SUMMARY +echo "ok" + +# 4b. Send .gcov files to appropriate path +echo -n "Sending coverage files to respective directories... " +for i in "$TMP"/*#*.gcov; do + mv $i $(echo $(basename $i) | sed 's/#/\//g') +done +echo "ok" diff --git a/testing/btest/Makefile b/testing/btest/Makefile index c9bcfff5ee..c6f2438ad1 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -21,6 +21,7 @@ coverage: cleanup: @rm -f $(DIAG) @rm -rf $(SCRIPT_COV)* + @find ../../ -name "*.gcda" -exec rm {} \; distclean: cleanup @rm -rf .btest.failed.dat \