diff --git a/.cirrus.yml b/.cirrus.yml index 2604595b92..c7cc780d9a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,7 +4,7 @@ btest_retries: &BTEST_RETRIES 2 memory: &MEMORY 4GB config: &CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install -sanitizer_config: &SANITIZER_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers +sanitizer_config: &SANITIZER_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers --enable-coverage resources_template: &RESOURCES_TEMPLATE cpu: *CPUS @@ -67,6 +67,9 @@ env: ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c] ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4] + # The repo token used for uploading data to Coveralls.io + ZEEK_COVERALLS_REPO_TOKEN: ENCRYPTED[7ffd1e041f848f02b62f5abc7fda8a5a8a1561fbb2b46d88cefb67c74408ddeef6ea6f3b279c7953ca14ae9b4d050e2d] + # Linux EOL timelines: https://linuxlifecycle.com/ # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle @@ -160,6 +163,7 @@ sanitizer_task: memory: 12GB << : *CI_TEMPLATE test_fuzzers_script: ./ci/test-fuzzers.sh + coverage_script: ./ci/upload-coverage.sh env: CXXFLAGS: -DZEEK_DICT_DEBUG ZEEK_CI_CONFIGURE_FLAGS: *SANITIZER_CONFIG diff --git a/CHANGES b/CHANGES index ba680b0d67..20a21d8178 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,12 @@ +3.3.0-dev.484 | 2020-10-23 14:47:44 -0700 + + * Add coveralls badge to README.md (Tim Wojtulewicz) + + * Remove the entire auxil directory from the results (Tim Wojtulewicz) + + * Prevent non-master builds, add cirrus information to upload (Tim Wojtulewicz) + + * Add support for sending data to coveralls with lcov_html.sh via Cirrus (Tim Wojtulewicz, Corelight) 3.3.0-dev.479 | 2020-10-23 12:25:15 -0700 diff --git a/README.md b/README.md index f9d8584738..4fc98685fa 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,11 @@ traffic analysis and security monitoring. Follow us on Twitter at [@zeekurity](https://twitter.com/zeekurity). +[![Coverage Status](https://coveralls.io/repos/github/zeek/zeek/badge.svg?branch=master)](https://coveralls.io/github/zeek/zeek?branch=master) + + Key Features -------------- diff --git a/VERSION b/VERSION index 73b43f3bb5..9635a29a9f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.0-dev.479 +3.3.0-dev.484 diff --git a/ci/ubuntu-18.04/Dockerfile b/ci/ubuntu-18.04/Dockerfile index 9f6276f95f..7905812721 100644 --- a/ci/ubuntu-18.04/Dockerfile +++ b/ci/ubuntu-18.04/Dockerfile @@ -22,6 +22,9 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ unzip \ + ruby \ + bc \ + lcov \ && rm -rf /var/lib/apt/lists/* # Many distros adhere to PEP 394's recommendation for `python` = `python2` so @@ -31,3 +34,4 @@ RUN ln -sf /usr/bin/python3 /usr/local/bin/python RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip RUN pip install junit2html +RUN gem install coveralls-lcov diff --git a/ci/upload-coverage.sh b/ci/upload-coverage.sh new file mode 100755 index 0000000000..250d8c90ef --- /dev/null +++ b/ci/upload-coverage.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env bash + +# The ZEEK_COVERALLS_REPO_TOKEN environment variable must exist +# for this script to work correctly. On Cirrus, this is provided +# via the secured variables. + +# Only do this on the master branch to avoid having a ton of data +# in Coveralls. +if [ "${CIRRUS_REPO_FULL_NAME}" != "zeek/zeek" ]; then + echo "Coverage upload skipped for non-zeek repo" + exit 0 +fi + +if [ "${CIRRUS_BRANCH}" != "master" ]; then + echo "Coverage upload skipped for non-master branches" + exit 0 +fi + +cd testing/coverage +make coverage +make coveralls diff --git a/testing/coverage/Makefile b/testing/coverage/Makefile index 7f458a4f9c..e621b2263d 100644 --- a/testing/coverage/Makefile +++ b/testing/coverage/Makefile @@ -9,4 +9,7 @@ distclean: cleanup @find ../../ -name "*.gcno" -exec rm {} \; html: - @./lcov_html.sh $(COVERAGE_HTML_DIR) + @./lcov_html.sh --html $(COVERAGE_HTML_DIR) + +coveralls: + @./lcov_html.sh --coveralls $(ZEEK_COVERALLS_REPO_TOKEN) diff --git a/testing/coverage/lcov_html.sh b/testing/coverage/lcov_html.sh index f17e583e2c..0d2c4e3854 100755 --- a/testing/coverage/lcov_html.sh +++ b/testing/coverage/lcov_html.sh @@ -7,28 +7,96 @@ # This depends on lcov to run. function die { - echo "$@" - exit 1 + echo "$@" + exit 1 } function finish { - rm -rf "$TMP" + rm -rf "$TMP" } function verify_run { - if bash -c "$1" > /dev/null 2>&1; then - echo ${2:-"ok"} - else - die ${3:-"error, abort"} - fi + if bash -c "$1" > /dev/null 2>&1; then + echo ${2:-"ok"} + else + die ${3:-"error, abort"} + fi } -trap finish EXIT +trap finish EXIT + +HTML_REPORT=1 +COVERALLS_REPO_TOKEN="" +COVERAGE_HTML_DIR="" + +function usage { + usage="\ +Usage: $0 + + Generate coverage data for the Zeek code. This uses data generated during btest, + so those should be run prior to calling this script. By default, this script + generates an HTML report in the coverage-html directory in the root of the Zeek + repo. + + Options: + --help Display this output. + --html DIR This is the default mode, but this argument can be passed + to make it explicit. It also can be used to pass an optional + destination directory for the HTML output. + --coveralls TOKEN Report coverage data to Coveralls.io using the specified + repo token. Enabling this option disables the HTML report. + This option requires the coveralls-lcov Ruby gem to be + installed. +" + + echo "${usage}" + exit 1 +} + +while (( "$#" )); do + case "$1" in + --html) + HTML_REPORT=1 + if [ ${#2} -eq 0 ]; then + COVERAGE_HTML_DIR="coverage-html" + shift 1 + else + COVERAGE_HTML_DIR=$2 + shift 2 + fi + ;; + --coveralls) + if [ ${#2} -eq 0 ]; then + echo "ERROR: Coveralls repo token must be passed with --coveralls argument." + echo + usage + fi + + HTML_REPORT=0 + COVERALLS_REPO_TOKEN=$2 + shift 2 + ;; + --help) + usage + shift 1 + ;; + *) + COVERAGE_HTML_DIR="${1:-"coverage-html"}" + shift 1 + ;; + esac +done TMP=".tmp.$$" COVERAGE_FILE="./$TMP/coverage.info" -COVERAGE_HTML_DIR="${1:-"coverage-html"}" -REMOVE_TARGETS="*.yy *.ll *.y *.l */bro.dir/* *.bif" + +if [ -z "${COVERAGE_HTML_DIR}" ]; then + COVERAGE_HTML_DIR="coverage-html" +fi + +# Files and directories that will be removed from the counts in step 5. Directories +# need to be surrounded by escaped wildcards. +REMOVE_TARGETS="*.yy *.ll *.y *.l \*/bro.dir/\* *.bif \*/zeek.dir/\* \*/src/3rdparty/\* \*/auxil/\*" # 1. Move to base dir, create tmp dir -cd ../../; +cd ../../; mkdir "$TMP" # 2. Check for .gcno and .gcda file presence @@ -41,21 +109,39 @@ for pat in gcda gcno; do done echo "ok" -# 3. If lcov does not exist, abort process. +# 3. If lcov does not exist, abort process. echo -n "Checking for lcov... " verify_run "which lcov" \ "lcov installed on system, continue" \ "lcov not installed, abort" -# 4. Create a "tracefile" through lcov, which is necessary to create html files later on. -echo -n "Creating tracefile for html generation... " +# 4. Create a "tracefile" through lcov, which is necessary to create output later on. +echo -n "Creating tracefile for output generation... " verify_run "lcov --no-external --capture --directory . --output-file $COVERAGE_FILE" +# 5. Remove a number of 3rdparty and "extra" files that shoudln't be included in the +# Zeek coverage numbers. for TARGET in $REMOVE_TARGETS; do - echo -n "Getting rid of $TARGET files from tracefile... " - verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE" + echo -n "Getting rid of $TARGET files from tracefile... " + verify_run "lcov --remove $COVERAGE_FILE $TARGET --output-file $COVERAGE_FILE" done -# 5. Create HTML files. -echo -n "Creating HTML files... " -verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE" +# 6. Create HTML files or Coveralls report +if [ $HTML_REPORT -eq 1 ]; then + echo -n "Creating HTML files... " + verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE" +else + echo -n "Reporting to Coveralls..." + coveralls_cmd="coveralls-lcov -t ${COVERALLS_REPO_TOKEN}" + + # If we're being called by Cirrus, add some additional information to the output. + if [ -n "${CIRRUS_BUILD_ID}" ]; then + coveralls_cmd="${coveralls_cmd} --service-name=cirrus --service-job-id=${CIRRUS_BUILD_ID}" + else + coveralls_cmd="${coveralls_cmd} --service-name=local" + fi + + coveralls_cmd="${coveralls_cmd} ${COVERAGE_FILE}" + + verify_run "${coveralls_cmd}" +fi