mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Merge branch 'master' of https://github.com/chungmin99/bro
* 'master' of https://github.com/chungmin99/bro: Renamed verify-run to verify_run Minor edits due to typo and field changes Added coverage to .PHONY in Makefile due to testing/coverage Fixing up `make html` target Refactoring, making error messages nicer, & lcov Add code coverage for bro source files after btest test suite
This commit is contained in:
Johanna Amann
commit
c95985f472
10 changed files with 265 additions and 2 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,2 +1,3 @@
|
|||
build
|
||||
tmp
|
||||
*.gcov
|
||||
|
|
|
|||
11
CHANGES
11
CHANGES
|
|
@ -1,4 +1,15 @@
|
|||
|
||||
2.5-826 | 2018-08-08 13:09:27 -0700
|
||||
|
||||
* Add support for code coverage statistics for bro source files after running btest
|
||||
test suite
|
||||
|
||||
This adds --enable-coverage flag to configure Bro with gcov.
|
||||
A new directory named /testing/coverage/ contains a new
|
||||
coverage target. By default a coverage.log is created; running
|
||||
make html in testing/coverage creates a HTML report.
|
||||
(Chung Min Kim, Corelight)
|
||||
|
||||
2.5-819 | 2018-08-08 13:03:22 -0500
|
||||
|
||||
* Fix cluster layout graphic and doc warnings (Jon Siwek, Corelight)
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
2.5-819
|
||||
2.5-826
|
||||
|
|
|
|||
7
configure
vendored
7
configure
vendored
|
|
@ -45,6 +45,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
|
|||
|
||||
Optional Features:
|
||||
--enable-debug compile in debugging mode (like --build-type=Debug)
|
||||
--enable-coverage compile with code coverage support (implies debugging mode)
|
||||
--enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275
|
||||
--enable-perftools force use of Google perftools on non-Linux systems
|
||||
(automatically on when perftools is present on Linux)
|
||||
|
|
@ -141,6 +142,8 @@ append_cache_entry INSTALL_BROCTL BOOL true
|
|||
append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
|
||||
append_cache_entry ENABLE_MOBILE_IPV6 BOOL false
|
||||
append_cache_entry DISABLE_PERFTOOLS BOOL false
|
||||
append_cache_entry DISABLE_RUBY_BINDINGS BOOL true
|
||||
append_cache_entry ENABLE_COVERAGE BOOL false
|
||||
|
||||
# parse arguments
|
||||
while [ $# -ne 0 ]; do
|
||||
|
|
@ -196,6 +199,10 @@ while [ $# -ne 0 ]; do
|
|||
--logdir=*)
|
||||
append_cache_entry BRO_LOG_DIR PATH $optarg
|
||||
;;
|
||||
--enable-coverage)
|
||||
append_cache_entry ENABLE_COVERAGE BOOL true
|
||||
append_cache_entry ENABLE_DEBUG BOOL true
|
||||
;;
|
||||
--enable-debug)
|
||||
append_cache_entry ENABLE_DEBUG BOOL true
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ brief: make-brief coverage
|
|||
distclean:
|
||||
@rm -f coverage.log
|
||||
$(MAKE) -C btest $@
|
||||
$(MAKE) -C coverage $@
|
||||
|
||||
make-verbose:
|
||||
@for repo in $(DIRS); do (cd $$repo && make -s ); done
|
||||
|
|
@ -22,4 +23,6 @@ coverage:
|
|||
@echo "Complete test suite code coverage:"
|
||||
@./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd`/../scripts
|
||||
@rm -f brocov.tmp.*
|
||||
@cd coverage && make coverage
|
||||
|
||||
.PHONY: coverage
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ coverage:
|
|||
cleanup:
|
||||
@rm -f $(DIAG)
|
||||
@rm -rf $(SCRIPT_COV)*
|
||||
@find ../../ -name "*.gcda" -exec rm {} \;
|
||||
|
||||
distclean: cleanup
|
||||
@rm -rf .btest.failed.dat \
|
||||
|
|
|
|||
12
testing/coverage/Makefile
Normal file
12
testing/coverage/Makefile
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
coverage: cleanup
|
||||
@./code_coverage.sh
|
||||
|
||||
cleanup:
|
||||
@rm -f coverage.log
|
||||
@find ../../ -name "*.gcov" -exec rm {} \;
|
||||
|
||||
distclean: cleanup
|
||||
@find ../../ -name "*.gcno" -exec rm {} \;
|
||||
|
||||
html:
|
||||
@./lcov_html.sh $(COVERAGE_HTML_DIR)
|
||||
21
testing/coverage/README
Normal file
21
testing/coverage/README
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
On a Bro build configured with --enable-coverage, this script produces a code
|
||||
coverage report after Bro has been invoked. The intended application of this
|
||||
script is after the btest testsuite has run. This combination (btests first,
|
||||
coverage computation afterward) happens automatically when running "make" in
|
||||
the testing directory. This script puts .gcov files (which are included in
|
||||
.gitignore) alongside the corresponding source files.
|
||||
|
||||
This depends on gcov, which should come with your gcc. If gcov is not
|
||||
installed, the script will abort with an error message.
|
||||
|
||||
After `make all` in the upper directory, use `make html` as make target in this
|
||||
directory to output the html files that lcov can create. By default, the html
|
||||
files will be contained in a directory named "coverage-html" in the base
|
||||
directory. To set a custom name, use `make html
|
||||
COVERAGE_HTML_DIR=custom-dir-name`.
|
||||
|
||||
The script code_coverage.sh is triggered by `make coverage` (included in `make`
|
||||
in /testing), and its goal is to automate code coverage testing.
|
||||
|
||||
The script lcov_html.sh is triggered by `make html`, and its goal is to create
|
||||
html files from the aforementioned coverage data.
|
||||
146
testing/coverage/code_coverage.sh
Executable file
146
testing/coverage/code_coverage.sh
Executable file
|
|
@ -0,0 +1,146 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# On a Bro build configured with --enable-coverage, this script
|
||||
# produces a code coverage report after Bro has been invoked. The
|
||||
# intended application of this script is after the btest testsuite has
|
||||
# run. This combination (btests first, coverage computation afterward)
|
||||
# happens automatically when running "make" in the testing directory.
|
||||
#
|
||||
# This depends on gcov, which should come with your gcc.
|
||||
#
|
||||
# AUTOMATES CODE COVERAGE TESTING
|
||||
# 1. Run test suite
|
||||
# 2. Check for .gcda files existing.
|
||||
# 3a. Run gcov (-p to preserve path)
|
||||
# 3b. Prune .gcov files for objects outside of the Bro tree
|
||||
# 4a. Analyze .gcov files generated and create summary file
|
||||
# 4b. Send .gcov files to appropriate path
|
||||
#
|
||||
CURR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Location of script
|
||||
BASE="$( cd "$CURR" && cd ../../ && pwd )"
|
||||
TMP="${CURR}/tmp.$$"
|
||||
mkdir -p $TMP
|
||||
|
||||
# DEFINE CLEANUP PROCESS
|
||||
function finish {
|
||||
rm -rf $TMP
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
# DEFINE CRUCIAL FUNCTIONS FOR COVERAGE CHECKING
|
||||
function check_file_coverage {
|
||||
GCOVDIR="$1"
|
||||
|
||||
for i in $GCOVDIR/*.gcov; do
|
||||
# Effective # of lines: starts with a number (# of runs in line) or ##### (line never run)
|
||||
TOTAL=$(cut -d: -f 1 "$i" | sed 's/ //g' | grep -v "^[[:alpha:]]" | grep -v "-" | wc -l)
|
||||
|
||||
# Count number of lines never run
|
||||
UNRUN=$(grep "#####" "$i" | wc -l)
|
||||
|
||||
# Lines in code are either run or unrun
|
||||
RUN=$(($TOTAL - $UNRUN))
|
||||
|
||||
# Avoid division-by-zero problems:
|
||||
PERCENTAGE=0.000
|
||||
[ $RUN -gt 0 ] && PERCENTAGE=$(bc <<< "scale=3; 100*$RUN/$TOTAL")
|
||||
|
||||
# Find correlation between % of lines run vs. "Runs"
|
||||
echo -e "$PERCENTAGE\t$RUN\t$TOTAL\t$(grep "0:Runs" "$i" | sed 's/.*://')\t$i"
|
||||
done
|
||||
}
|
||||
|
||||
function check_group_coverage {
|
||||
DATA="$1" # FILE CONTAINING COVERAGE DATA
|
||||
SRC_FOLDER="$2" # WHERE BRO WAS COMPILED
|
||||
OUTPUT="$3"
|
||||
|
||||
# Prints all the relevant directories
|
||||
DIRS=$(for i in $(cut -f 5 "$DATA"); do basename "$i" | sed 's/#[^#]*$//'; done \
|
||||
| sort | uniq | sed 's/^.*'"${SRC_FOLDER}"'//' | grep "^#s\+" )
|
||||
# "Generalize" folders unless it's from analyzers
|
||||
DIRS=$(for i in $DIRS; do
|
||||
if !(echo "$i" | grep "src#analyzer"); then
|
||||
echo "$i" | cut -d "#" -f 1,2,3
|
||||
fi
|
||||
done | sort | uniq )
|
||||
|
||||
for i in $DIRS; do
|
||||
# For elements in #src, we only care about the files direclty in the directory.
|
||||
if [[ "$i" = "#src" ]]; then
|
||||
RUN=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 2) | tr " " "+" | bc)
|
||||
TOTAL=$(echo $(grep "$i#[^#]\+$" $DATA | grep "$SRC_FOLDER$i\|build$i" | cut -f 3) | tr " " "+" | bc)
|
||||
else
|
||||
RUN=$(echo $(grep "$i" $DATA | cut -f 2) | tr " " "+" | bc)
|
||||
TOTAL=$(echo $(grep "$i" $DATA | cut -f 3) | tr " " "+" | bc)
|
||||
fi
|
||||
|
||||
PERCENTAGE=$( echo "scale=3;100*$RUN/$TOTAL" | bc | tr "\n" " " )
|
||||
printf "%-50s\t%12s\t%6s %%\n" "$i" "$RUN/$TOTAL" $PERCENTAGE \
|
||||
| sed 's|#|/|g' >>$OUTPUT
|
||||
done
|
||||
}
|
||||
|
||||
# 1. Run test suite
|
||||
# SHOULD HAVE ALREADY BEEN RUN BEFORE THIS SCRIPT (BASED ON MAKEFILE TARGETS)
|
||||
|
||||
# 2. Check for .gcno and .gcda file presence
|
||||
echo -n "Checking for coverage files... "
|
||||
for pat in gcda gcno; do
|
||||
if [ -z "$(find "$BASE" -name "*.$pat" 2>/dev/null)" ]; then
|
||||
echo "no .$pat files, nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "ok"
|
||||
|
||||
# 3a. Run gcov (-p to preserve path) and move into tmp directory
|
||||
# ... if system does not have gcov installed, exit with message.
|
||||
echo -n "Creating coverage files... "
|
||||
if which gcov > /dev/null 2>&1; then
|
||||
( cd "$TMP" && find "$BASE" -name "*.o" -exec gcov -p {} > /dev/null 2>&1 \; )
|
||||
NUM_GCOVS=$(find "$TMP" -name *.gcov | wc -l)
|
||||
if [ $NUM_GCOVS -eq 0 ]; then
|
||||
echo "no gcov files produced, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Account for '^' that occurs in macOS due to LLVM
|
||||
# This character seems to be equivalent to ".." (up 1 dir)
|
||||
for file in $(ls $TMP/*.gcov | grep '\^'); do
|
||||
mv $file "$(sed 's/#[^#]*#\^//g' <<< "$file")"
|
||||
done
|
||||
|
||||
echo "ok, $NUM_GCOVS coverage files"
|
||||
else
|
||||
echo "gcov is not installed on system, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3b. Prune gcov files that fall outside of the Bro tree:
|
||||
# Look for files containing gcov's slash substitution character "#"
|
||||
# and remove any that don't contain the Bro path root.
|
||||
echo -n "Pruning out-of-tree coverage files... "
|
||||
PREFIX=$(echo "$BASE" | sed 's|/|#|g')
|
||||
for i in "$TMP"/*#*.gcov; do
|
||||
if ! [[ "$i" = *$PREFIX* ]]; then
|
||||
rm -f $i
|
||||
fi
|
||||
done
|
||||
NUM_GCOVS=$(ls "$TMP"/*.gcov | wc -l)
|
||||
echo "ok, $NUM_GCOVS coverage files remain"
|
||||
|
||||
# 4a. Analyze .gcov files generated and create summary file
|
||||
echo -n "Creating summary file... "
|
||||
DATA="${TMP}/data.txt"
|
||||
SUMMARY="$CURR/coverage.log"
|
||||
check_file_coverage "$TMP" > "$DATA"
|
||||
check_group_coverage "$DATA" ${BASE##*/} $SUMMARY
|
||||
echo "ok"
|
||||
|
||||
# 4b. Send .gcov files to appropriate path
|
||||
echo -n "Sending coverage files to respective directories... "
|
||||
for i in "$TMP"/*#*.gcov; do
|
||||
mv $i $(echo $(basename $i) | sed 's/#/\//g')
|
||||
done
|
||||
echo "ok"
|
||||
61
testing/coverage/lcov_html.sh
Executable file
61
testing/coverage/lcov_html.sh
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# On a Bro build configured with --enable-coverage, this script
|
||||
# produces a code coverage report in HTML format after Bro has been invoked. The
|
||||
# intended application of this script is after the btest testsuite has run.
|
||||
|
||||
# This depends on lcov to run.
|
||||
|
||||
function die {
|
||||
echo "$@"
|
||||
exit 1
|
||||
}
|
||||
function finish {
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
function verify_run {
|
||||
if bash -c "$1" > /dev/null 2>&1; then
|
||||
echo ${2:-"ok"}
|
||||
else
|
||||
die ${3:-"error, abort"}
|
||||
fi
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
TMP=".tmp.$$"
|
||||
COVERAGE_FILE="./$TMP/coverage.info"
|
||||
COVERAGE_HTML_DIR="${1:-"coverage-html"}"
|
||||
REMOVE_TARGETS="*.yy *.ll *.y *.l */bro.dir/* *.bif"
|
||||
|
||||
# 1. Move to base dir, create tmp dir
|
||||
cd ../../;
|
||||
mkdir "$TMP"
|
||||
|
||||
# 2. Check for .gcno and .gcda file presence
|
||||
echo -n "Checking for coverage files... "
|
||||
for pat in gcda gcno; do
|
||||
if [ -z "$(find . -name "*.$pat" 2>/dev/null)" ]; then
|
||||
echo "no .$pat files, nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "ok"
|
||||
|
||||
# 3. If lcov does not exist, abort process.
|
||||
echo -n "Checking for lcov... "
|
||||
verify_run "which lcov" \
|
||||
"lcov installed on system, continue" \
|
||||
"lcov not installed, abort"
|
||||
|
||||
# 4. Create a "tracefile" through lcov, which is necessary to create html files later on.
|
||||
echo -n "Creating tracefile for html generation... "
|
||||
verify_run "lcov --no-external --capture --directory . --output-file $COVERAGE_FILE"
|
||||
|
||||
for TARGET in $REMOVE_TARGETS; do
|
||||
echo -n "Getting rid of $TARGET files from tracefile... "
|
||||
verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE"
|
||||
done
|
||||
|
||||
# 5. Create HTML files.
|
||||
echo -n "Creating HTML files... "
|
||||
verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE"
|
||||
Loading…
Add table
Add a link
Reference in a new issue