Merge branch 'master' of https://github.com/chungmin99/bro

* 'master' of https://github.com/chungmin99/bro:
  Renamed verify-run to verify_run
  Minor edits due to typo and field changes
  Added coverage to .PHONY in Makefile due to testing/coverage
  Fixing up `make html` target
  Refactoring, making error messages nicer, & lcov
  Add code coverage for bro source files after btest test suite

.gitignore

@@ -1,2 +1,3 @@
 build
 tmp
+*.gcov

CHANGES

@@ -1,4 +1,15 @@
 
+2.5-826 | 2018-08-08 13:09:27 -0700
+
+  * Add support for code coverage statistics for bro source files after running btest
+    test suite
+
+    This adds --enable-coverage flag to configure Bro with gcov.
+    A new directory named /testing/coverage/ contains a new
+    coverage target. By default a coverage.log is created; running
+    make html in testing/coverage creates a HTML report.
+    (Chung Min Kim, Corelight)
+
 2.5-819 | 2018-08-08 13:03:22 -0500
 
   * Fix cluster layout graphic and doc warnings (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-2.5-819
+2.5-826

configure

@@ -45,6 +45,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Optional Features:
     --enable-debug         compile in debugging mode (like --build-type=Debug)
+    --enable-coverage      compile with code coverage support (implies debugging mode)
     --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
     --enable-perftools     force use of Google perftools on non-Linux systems
                            (automatically on when perftools is present on Linux)
@@ -141,6 +142,8 @@ append_cache_entry INSTALL_BROCTL       BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry DISABLE_PERFTOOLS    BOOL   false
+append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   true
+append_cache_entry ENABLE_COVERAGE	BOOL   false
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -196,6 +199,10 @@ while [ $# -ne 0 ]; do
         --logdir=*)
             append_cache_entry BRO_LOG_DIR  PATH   $optarg
             ;;
+        --enable-coverage)
+            append_cache_entry ENABLE_COVERAGE         BOOL   true
+            append_cache_entry ENABLE_DEBUG         BOOL   true
+            ;;
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true
             ;;

testing/Makefile

@@ -8,6 +8,7 @@ brief: make-brief coverage
 distclean:
 	@rm -f coverage.log
 	$(MAKE) -C btest $@
+	$(MAKE) -C coverage $@
 
 make-verbose:
 	@for repo in $(DIRS); do (cd $$repo && make -s ); done
@@ -22,4 +23,6 @@ coverage:
 	@echo "Complete test suite code coverage:"
 	@./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd`/../scripts
 	@rm -f brocov.tmp.*
+	@cd coverage && make coverage
 
+.PHONY: coverage

testing/btest/Makefile

@@ -21,6 +21,7 @@ coverage:
 cleanup:
 	@rm -f $(DIAG)
 	@rm -rf $(SCRIPT_COV)*
+	@find ../../ -name "*.gcda" -exec rm {} \;
 
 distclean: cleanup
 	@rm -rf .btest.failed.dat \

testing/coverage/Makefile

@@ -0,0 +1,12 @@
+coverage: cleanup
+	@./code_coverage.sh
+
+cleanup:
+	@rm -f coverage.log
+	@find ../../ -name "*.gcov" -exec rm {} \;
+
+distclean: cleanup
+	@find ../../ -name "*.gcno" -exec rm {} \;
+
+html:
+	@./lcov_html.sh $(COVERAGE_HTML_DIR)

testing/coverage/README

@@ -0,0 +1,21 @@
+On a Bro build configured with --enable-coverage, this script produces a code
+coverage report after Bro has been invoked. The intended application of this
+script is after the btest testsuite has run. This combination (btests first,
+coverage computation afterward) happens automatically when running "make" in
+the testing directory.  This script puts .gcov files (which are included in
+.gitignore) alongside the corresponding source files.
+
+This depends on gcov, which should come with your gcc. If gcov is not
+installed, the script will abort with an error message.
+
+After `make all` in the upper directory, use `make html` as make target in this
+directory to output the html files that lcov can create. By default, the html
+files will be contained in a directory named "coverage-html" in the base
+directory. To set a custom name, use `make html
+COVERAGE_HTML_DIR=custom-dir-name`.
+
+The script code_coverage.sh is triggered by `make coverage` (included in `make`
+in /testing), and its goal is to automate code coverage testing.
+
+The script lcov_html.sh is triggered by `make html`, and its goal is to create
+html files from the aforementioned coverage data.

testing/coverage/code_coverage.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+#
+# On a Bro build configured with --enable-coverage, this script
+# produces a code coverage report after Bro has been invoked. The
+# intended application of this script is after the btest testsuite has
+# run. This combination (btests first, coverage computation afterward)
+# happens automatically when running "make" in the testing directory.
+#
+# This depends on gcov, which should come with your gcc.
+#
+# AUTOMATES CODE COVERAGE TESTING
+#	1. Run test suite
+# 	2. Check for .gcda files existing.
+#	3a. Run gcov (-p to preserve path)
+#       3b. Prune .gcov files for objects outside of the Bro tree
+#	4a. Analyze .gcov files generated and create summary file
+#	4b. Send .gcov files to appropriate path
+#
+CURR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Location of script
+BASE="$( cd "$CURR" && cd ../../ && pwd )"
+TMP="${CURR}/tmp.$$"
+mkdir -p $TMP
+
+# DEFINE CLEANUP PROCESS
+function finish {
+	rm -rf $TMP
+}
+trap finish EXIT
+
+# DEFINE CRUCIAL FUNCTIONS FOR COVERAGE CHECKING
+function check_file_coverage {
+	GCOVDIR="$1"
+
+	for i in $GCOVDIR/*.gcov; do
+		# Effective # of lines: starts with a number (# of runs in line) or ##### (line never run)
+		TOTAL=$(cut -d: -f 1 "$i" | sed 's/ //g' | grep -v "^[[:alpha:]]" | grep -v "-" | wc -l)
+
+		# Count number of lines never run
+		UNRUN=$(grep "#####" "$i" | wc -l)
+
+		# Lines in code are either run or unrun
+		RUN=$(($TOTAL - $UNRUN))
+
+                # Avoid division-by-zero problems:
+                PERCENTAGE=0.000
+                [ $RUN -gt 0 ] && PERCENTAGE=$(bc <<< "scale=3; 100*$RUN/$TOTAL")
+
+		# Find correlation between % of lines run vs. "Runs"
+		echo -e "$PERCENTAGE\t$RUN\t$TOTAL\t$(grep "0:Runs" "$i" | sed 's/.*://')\t$i"
+	done
+}
+
+function check_group_coverage {
+	DATA="$1"	# FILE CONTAINING COVERAGE DATA
+	SRC_FOLDER="$2" # WHERE BRO WAS COMPILED
+	OUTPUT="$3"
+
+	# Prints all the relevant directories
+	DIRS=$(for i in $(cut -f 5 "$DATA"); do basename "$i" | sed 's/#[^#]*$//'; done \
+		| sort | uniq | sed 's/^.*'"${SRC_FOLDER}"'//' | grep "^#s\+" )
+	# "Generalize" folders unless it's from analyzers
+	DIRS=$(for i in $DIRS; do
+		if !(echo "$i" | grep "src#analyzer"); then
+			echo "$i" | cut -d "#" -f 1,2,3
+		fi
+	done | sort | uniq )
+
+	for i in $DIRS; do
+		# For elements in #src, we only care about the files direclty in the directory.
+		if [[ "$i" = "#src" ]]; then
+			RUN=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 2) | tr " " "+" | bc)
+			TOTAL=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 3) | tr " " "+" | bc)
+		else
+			RUN=$(echo $(grep "$i" $DATA | cut -f 2) | tr " " "+" | bc)
+			TOTAL=$(echo $(grep "$i" $DATA | cut -f 3) | tr " " "+" | bc)
+		fi
+
+		PERCENTAGE=$( echo "scale=3;100*$RUN/$TOTAL" | bc | tr "\n" " " )
+		printf "%-50s\t%12s\t%6s %%\n" "$i" "$RUN/$TOTAL" $PERCENTAGE \
+                       | sed 's|#|/|g' >>$OUTPUT
+	done
+}
+
+# 1. Run test suite
+# SHOULD HAVE ALREADY BEEN RUN BEFORE THIS SCRIPT (BASED ON MAKEFILE TARGETS)
+
+# 2. Check for .gcno and .gcda file presence
+echo -n "Checking for coverage files... "
+for pat in gcda gcno; do
+    if [ -z "$(find "$BASE" -name "*.$pat" 2>/dev/null)" ]; then
+        echo "no .$pat files, nothing to do"
+	exit 0
+    fi
+done
+echo "ok"
+
+# 3a. Run gcov (-p to preserve path) and move into tmp directory
+# ... if system does not have gcov installed, exit with message.
+echo -n "Creating coverage files... "
+if which gcov > /dev/null 2>&1; then
+	( cd "$TMP" && find "$BASE" -name "*.o" -exec gcov -p {} > /dev/null 2>&1 \; )
+	NUM_GCOVS=$(find "$TMP" -name *.gcov | wc -l)
+	if [ $NUM_GCOVS -eq 0 ]; then
+		echo "no gcov files produced, aborting"
+		exit 1
+	fi
+
+	# Account for '^' that occurs in macOS due to LLVM
+	# This character seems to be equivalent to ".." (up 1 dir)
+	for file in $(ls $TMP/*.gcov | grep '\^'); do
+		mv $file "$(sed 's/#[^#]*#\^//g' <<< "$file")"
+	done
+
+	echo "ok, $NUM_GCOVS coverage files"
+else
+	echo "gcov is not installed on system, aborting"
+	exit 1
+fi
+
+# 3b. Prune gcov files that fall outside of the Bro tree:
+# Look for files containing gcov's slash substitution character "#"
+# and remove any that don't contain the Bro path root.
+echo -n "Pruning out-of-tree coverage files... "
+PREFIX=$(echo "$BASE" | sed 's|/|#|g')
+for i in "$TMP"/*#*.gcov; do
+    if ! [[ "$i" = *$PREFIX* ]]; then
+        rm -f $i
+    fi
+done
+NUM_GCOVS=$(ls "$TMP"/*.gcov | wc -l)
+echo "ok, $NUM_GCOVS coverage files remain"
+
+# 4a. Analyze .gcov files generated and create summary file
+echo -n "Creating summary file... "
+DATA="${TMP}/data.txt"
+SUMMARY="$CURR/coverage.log"
+check_file_coverage "$TMP" > "$DATA"
+check_group_coverage "$DATA" ${BASE##*/} $SUMMARY
+echo "ok"
+
+# 4b. Send .gcov files to appropriate path
+echo -n "Sending coverage files to respective directories... "
+for i in "$TMP"/*#*.gcov; do
+    mv $i $(echo $(basename $i) | sed 's/#/\//g')
+done
+echo "ok"

testing/coverage/lcov_html.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# On a Bro build configured with --enable-coverage, this script
+# produces a code coverage report in HTML format after Bro has been invoked. The
+# intended application of this script is after the btest testsuite has run.
+
+# This depends on lcov to run.
+
+function die {
+	echo "$@"
+	exit 1
+}
+function finish {
+	rm -rf "$TMP"
+}
+function verify_run {
+	if bash -c "$1" > /dev/null 2>&1; then
+		echo ${2:-"ok"}
+	else
+		die ${3:-"error, abort"}
+	fi
+}
+trap finish EXIT 
+
+TMP=".tmp.$$"
+COVERAGE_FILE="./$TMP/coverage.info"
+COVERAGE_HTML_DIR="${1:-"coverage-html"}"
+REMOVE_TARGETS="*.yy *.ll *.y *.l */bro.dir/* *.bif"
+
+# 1. Move to base dir, create tmp dir
+cd ../../; 
+mkdir "$TMP"
+
+# 2. Check for .gcno and .gcda file presence
+echo -n "Checking for coverage files... "
+for pat in gcda gcno; do
+    if [ -z "$(find . -name "*.$pat" 2>/dev/null)" ]; then
+        echo "no .$pat files, nothing to do"
+	exit 0
+    fi
+done
+echo "ok"
+
+# 3. If lcov does not exist, abort process. 
+echo -n "Checking for lcov... "
+verify_run "which lcov" \
+	"lcov installed on system, continue" \
+	"lcov not installed, abort"
+
+# 4. Create a "tracefile" through lcov, which is necessary to create html files later on.
+echo -n "Creating tracefile for html generation... " 
+verify_run "lcov --no-external --capture --directory . --output-file $COVERAGE_FILE"
+
+for TARGET in $REMOVE_TARGETS; do
+	echo -n "Getting rid of $TARGET files from tracefile... "
+	verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE"
+done
+
+# 5. Create HTML files. 
+echo -n "Creating HTML files... "
+verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE"