Add support for sending data to coveralls with lcov_html.sh via Cirrus

.cirrus.yml

@@ -4,7 +4,7 @@ btest_retries: &BTEST_RETRIES 2
 memory: &MEMORY 4GB
 
 config: &CONFIG --build-type=release --enable-cpp-tests  --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
-sanitizer_config: &SANITIZER_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers
+sanitizer_config: &SANITIZER_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers --enable-coverage
 
 resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS
@@ -67,6 +67,9 @@ env:
   ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c]
   ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4]
 
+  # The repo token used for uploading data to Coveralls.io
+  ZEEK_COVERALLS_REPO_TOKEN: ENCRYPTED[7ffd1e041f848f02b62f5abc7fda8a5a8a1561fbb2b46d88cefb67c74408ddeef6ea6f3b279c7953ca14ae9b4d050e2d]
+
 # Linux EOL timelines: https://linuxlifecycle.com/
 # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
 
@@ -160,6 +163,7 @@ sanitizer_task:
     memory: 12GB
   << : *CI_TEMPLATE
   test_fuzzers_script: ./ci/test-fuzzers.sh
+  coverage_script: ./ci/upload-coverage.sh
   env:
     CXXFLAGS: -DZEEK_DICT_DEBUG
     ZEEK_CI_CONFIGURE_FLAGS: *SANITIZER_CONFIG

ci/ubuntu-18.04/Dockerfile

@@ -22,6 +22,9 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     unzip \
+    ruby \
+    bc \
+    lcov \
   && rm -rf /var/lib/apt/lists/*
 
 # Many distros adhere to PEP 394's recommendation for `python` = `python2` so
@@ -31,3 +34,4 @@ RUN ln -sf /usr/bin/python3 /usr/local/bin/python
 RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
 
 RUN pip install junit2html
+RUN gem install coveralls-lcov

ci/upload-coverage.sh

@@ -0,0 +1,21 @@
+#! /usr/bin/env bash
+
+# The ZEEK_COVERALLS_REPO_TOKEN environment variable must exist
+# for this script to work correctly. On Cirrus, this is provided
+# via the secured variables.
+
+# Only do this on the master branch to avoid having a ton of data
+# in Coveralls.
+if [ "${CIRRUS_REPO_FULL_NAME}" != "zeek/zeek" ]; then
+    echo "Coverage upload skipped for non-zeek repo"
+    exit 0
+fi
+
+if [ "${CIRRUS_BRANCH}" != "master" ] then
+   echo "Coverage upload skipped for non-master branches"
+   exit 0
+fi
+
+cd testing/coverage
+make coverage
+make coveralls

testing/coverage/Makefile

@@ -9,4 +9,7 @@ distclean: cleanup
 	@find ../../ -name "*.gcno" -exec rm {} \;
 
 html:
-	@./lcov_html.sh $(COVERAGE_HTML_DIR)
+	@./lcov_html.sh --html $(COVERAGE_HTML_DIR)
+
+coveralls:
+	@./lcov_html.sh --coveralls $(ZEEK_COVERALLS_REPO_TOKEN)

testing/coverage/lcov_html.sh

@@ -7,25 +7,93 @@
 # This depends on lcov to run.
 
 function die {
-	echo "$@"
+    echo "$@"
-	exit 1
+    exit 1
 }
 function finish {
-	rm -rf "$TMP"
+    rm -rf "$TMP"
 }
 function verify_run {
-	if bash -c "$1" > /dev/null 2>&1; then
+    if bash -c "$1" > /dev/null 2>&1; then
-		echo ${2:-"ok"}
+	echo ${2:-"ok"}
-	else
+    else
-		die ${3:-"error, abort"}
+	die ${3:-"error, abort"}
-	fi
+    fi
 }
 trap finish EXIT
 
+HTML_REPORT=1
+COVERALLS_REPO_TOKEN=""
+COVERAGE_HTML_DIR=""
+
+function usage {
+    usage="\
+Usage: $0 <options>
+
+  Generate coverage data for the Zeek code. This uses data generated during btest,
+  so those should be run prior to calling this script. By default, this script
+  generates an HTML report in the coverage-html directory in the root of the Zeek
+  repo.
+
+  Options:
+    --help             Display this output.
+    --html DIR         This is the default mode, but this argument can be passed
+                       to make it explicit. It also can be used to pass an optional
+                       destination directory for the HTML output.
+    --coveralls TOKEN  Report coverage data to Coveralls.io using the specified
+                       repo token. Enabling this option disables the HTML report.
+                       This option requires the coveralls-lcov Ruby gem to be
+                       installed.
+"
+
+    echo "${usage}"
+    exit 1
+}
+
+while (( "$#" )); do
+    case "$1" in
+	--html)
+	    HTML_REPORT=1
+	    if [ ${#2} -eq 0 ]; then
+		COVERAGE_HTML_DIR="coverage-html"
+		shift 1
+	    else
+		COVERAGE_HTML_DIR=$2
+		shift 2
+	    fi
+	    ;;
+	--coveralls)
+	    if [ ${#2} -eq 0 ]; then
+		echo "ERROR: Coveralls repo token must be passed with --coveralls argument."
+		echo
+		usage
+	    fi
+
+	    HTML_REPORT=0
+	    COVERALLS_REPO_TOKEN=$2
+	    shift 2
+	    ;;
+	--help)
+	    usage
+	    shift 1
+	    ;;
+	*)
+	    COVERAGE_HTML_DIR="${1:-"coverage-html"}"
+	    shift 1
+	    ;;
+    esac
+done
+
 TMP=".tmp.$$"
 COVERAGE_FILE="./$TMP/coverage.info"
-COVERAGE_HTML_DIR="${1:-"coverage-html"}"
+
-REMOVE_TARGETS="*.yy *.ll *.y *.l */bro.dir/* *.bif"
+if [ -z "${COVERAGE_HTML_DIR}" ]; then
+    COVERAGE_HTML_DIR="coverage-html"
+fi
+
+# Files and directories that will be removed from the counts in step 5. Directories
+# need to be surrounded by escaped wildcards.
+REMOVE_TARGETS="*.yy *.ll *.y *.l \*/bro.dir/\* *.bif \*/zeek.dir/\* \*/rapidjson/\* \*/highwayhash/\* \*/caf/\* \*/src/3rdparty/\* \*/broker/3rdparty/\* \*/auxil/bifcl/\* \*/auxil/binpac/\*"
 
 # 1. Move to base dir, create tmp dir
 cd ../../;
@@ -51,11 +119,18 @@ verify_run "which lcov" \
 echo -n "Creating tracefile for html generation... "
 verify_run "lcov --no-external --capture --directory . --output-file $COVERAGE_FILE"
 
+# 5. Remove a number of 3rdparty and "extra" files that shoudln't be included in the
+# Zeek coverage numbers.
 for TARGET in $REMOVE_TARGETS; do
-	echo -n "Getting rid of $TARGET files from tracefile... "
+    echo -n "Getting rid of $TARGET files from tracefile... "
-	verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE"
+    verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE"
 done
 
-# 5. Create HTML files. 
+# 6. Create HTML files or Coveralls report
-echo -n "Creating HTML files... "
+if [ $HTML_REPORT -eq 1 ]; then
-verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE"
+    echo -n "Creating HTML files... "
+    verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE"
+else
+    echo -n "Reporting to Coveralls..."
+    verify_run "coveralls-lcov -t ${COVERALLS_REPO_TOKEN} ${COVERAGE_FILE}"
+fi