From 80e3c42526d441affd0778007a99daac6c9cf141 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 8 Aug 2019 10:01:03 -0700 Subject: [PATCH 1/2] Add new distro to Travis CI configuration for running leak tests --- .travis.yml | 1 + testing/scripts/travis-job | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a1bb0207ee..e10a8d5ffc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ env: - distro: fedora_28 - distro: ubuntu_16.04 - distro: ubuntu_18.04 + - distro: ubuntu_18.04_leaktest install: sh testing/scripts/travis-job install $distro diff --git a/testing/scripts/travis-job b/testing/scripts/travis-job index 4b15166a44..a166f04c3f 100644 --- a/testing/scripts/travis-job +++ b/testing/scripts/travis-job @@ -13,6 +13,7 @@ usage() { echo " CMD is a build step:" echo " install: install prereqs" echo " build: build zeek" + echo " build_leaktests: build zeek with memory leak testing enabled" echo " run: run the tests" echo " all: do all of the above" echo " DISTRO is a Linux distro, 'travis' to run without docker, or 'coverity' to run a coverity scan" @@ -29,6 +30,7 @@ distro=$2 case $step in install) ;; build) ;; + build_leaktests) ;; run) ;; all) ;; *) echo "Error: unknown build step: $step"; usage; exit 1 ;; @@ -78,6 +80,7 @@ run_coverity() { # Create a docker container, and install all packages needed to build Zeek. install_in_docker() { + local_distro=$distro case $distro in centos_7) distro_cmds="yum -y install gdb cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel git openssl which" @@ -94,13 +97,17 @@ install_in_docker() { ubuntu_18.04) distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils; ln -s /usr/bin/python3 /usr/local/bin/python" ;; + ubuntu_18.04_leaktest) + distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils google-perftools libgoogle-perftools4 libgoogle-perftools-dev; ln -s /usr/bin/python3 /usr/local/bin/python" + local_distro="ubuntu_18.04" + ;; *) echo "Error: distro ${distro} is not recognized by this script" exit 1 ;; esac - docker_image=`echo $distro | tr '_' ':'` + docker_image=`echo $local_distro | tr '_' ':'` docker run --name zeektest -id -v "`pwd`:/zeek" -w /zeek ${docker_image} sh docker exec zeektest sh -c "${distro_cmds}" } @@ -108,7 +115,11 @@ install_in_docker() { # Build Zeek in a docker container. build_in_docker() { - docker exec zeektest sh testing/scripts/travis-job build travis + if [ "${distro}" = "ubuntu_18.04_leaktest" ]; then + docker exec zeektest sh testing/scripts/travis-job build_leaktests travis + else + docker exec zeektest sh testing/scripts/travis-job build travis + fi } @@ -126,9 +137,21 @@ build() { # Skip building Broker tests, python bindings, and zeekctl, as these are # not needed by the Zeek tests. + echo "Configuring for normal build" ./configure --build-type=Release --disable-broker-tests --disable-python --disable-zeekctl && make -j 2 } +# Build Zeek for leak testing +build_leaktests() { + # Cleanup any previous build (this is really only necessary if running this + # outside of Travis). + make distclean > /dev/null + + # Skip building Broker tests, python bindings, and zeekctl, as these are + # not needed by the Zeek tests. + echo "Configuring with perftools for leak testing" + ./configure --build-type=Debug --disable-broker-tests --disable-python --disable-zeekctl --enable-perftools --enable-perftools-debug && make -j 2 +} # Get the private tests. get_private_tests() { @@ -336,6 +359,8 @@ elif [ "$distro" = "travis" ]; then # a single line when viewing the "Job log" on the Travis CI web site). if [ "$step" = "build" ]; then build + elif [ "$step" = "build_leaktests" ]; then + build_leaktests elif [ "$step" = "run" ]; then run elif [ "$step" = "all" ]; then From b9f6992c0eed094cb0faea5c8c0a63fd7913d085 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 9 Aug 2019 11:01:16 -0700 Subject: [PATCH 2/2] fixup! Add new distro to Travis CI configuration for running leak tests --- testing/external/commit-hash.zeek-testing | 2 +- .../external/commit-hash.zeek-testing-private | 2 +- testing/scripts/travis-job | 85 ++++++++++--------- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 9ac9662c8e..aa7668640c 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -4e78e7e6f9baf56ec6303d2580f380628fd31e36 +f09295557a33f20bd36415d100964ce5cadd8077 diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index b69a84b06b..a7c2f6797a 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -6fef8f0d8eb896856fa72551af65668d0ea01bf4 +faf8fa15c30a116b5401b03044db6eecfa4dd693 diff --git a/testing/scripts/travis-job b/testing/scripts/travis-job index a166f04c3f..4c87925f33 100644 --- a/testing/scripts/travis-job +++ b/testing/scripts/travis-job @@ -8,6 +8,8 @@ # outside of Travis then you will need to fetch the private tests manually # (if you don't, then the private tests will be skipped). +LEAK_TEST_DISTRO="ubuntu_18.04_leaktest" + usage() { echo "usage: $0 CMD DISTRO" echo " CMD is a build step:" @@ -30,7 +32,6 @@ distro=$2 case $step in install) ;; build) ;; - build_leaktests) ;; run) ;; all) ;; *) echo "Error: unknown build step: $step"; usage; exit 1 ;; @@ -97,7 +98,7 @@ install_in_docker() { ubuntu_18.04) distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils; ln -s /usr/bin/python3 /usr/local/bin/python" ;; - ubuntu_18.04_leaktest) + ${LEAK_TEST_DISTRO}) distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils google-perftools libgoogle-perftools4 libgoogle-perftools-dev; ln -s /usr/bin/python3 /usr/local/bin/python" local_distro="ubuntu_18.04" ;; @@ -115,17 +116,17 @@ install_in_docker() { # Build Zeek in a docker container. build_in_docker() { - if [ "${distro}" = "ubuntu_18.04_leaktest" ]; then - docker exec zeektest sh testing/scripts/travis-job build_leaktests travis - else - docker exec zeektest sh testing/scripts/travis-job build travis - fi + # Pass the distro as a different environment variable name to docker since + # the script will set $distro to "travis" as part of the invocation. + docker exec -e BUILD_DISTRO=${distro} zeektest sh testing/scripts/travis-job build travis } # Run Zeek tests in a docker container. run_in_docker() { - docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e TESTING_PRIVATE_DEPLOYKEY zeektest sh testing/scripts/travis-job run travis + # Pass the distro as a different environment variable name to docker since + # the script will set $distro to "travis" as part of the invocation. + docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e TESTING_PRIVATE_DEPLOYKEY -e BUILD_DISTRO=${distro} zeektest sh testing/scripts/travis-job run travis } @@ -136,21 +137,14 @@ build() { make distclean > /dev/null # Skip building Broker tests, python bindings, and zeekctl, as these are - # not needed by the Zeek tests. - echo "Configuring for normal build" - ./configure --build-type=Release --disable-broker-tests --disable-python --disable-zeekctl && make -j 2 -} - -# Build Zeek for leak testing -build_leaktests() { - # Cleanup any previous build (this is really only necessary if running this - # outside of Travis). - make distclean > /dev/null - - # Skip building Broker tests, python bindings, and zeekctl, as these are - # not needed by the Zeek tests. - echo "Configuring with perftools for leak testing" - ./configure --build-type=Debug --disable-broker-tests --disable-python --disable-zeekctl --enable-perftools --enable-perftools-debug && make -j 2 + # not needed by the Zeek tests. If the distro is set for leak tests, enable + # those options as well. + if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then + ./configure --build-type=Release --disable-broker-tests --disable-python --disable-zeekctl && make -j 2 + else + echo "Configuring zeek to build for leak testing" + ./configure --build-type=Debug --disable-broker-tests --disable-python --disable-zeekctl --enable-perftools --enable-perftools-debug && make -j 2 + fi } # Get the private tests. @@ -193,7 +187,12 @@ run() { set +e # Must specify a value for "-j" option, otherwise Travis uses a huge value. - ../../aux/btest/btest -j 4 -d + if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then + ../../aux/btest/btest -j 4 -d + else + ../../aux/btest/btest -j 4 -d -g leaks + fi + ret=$? set -e @@ -231,27 +230,35 @@ run() { set +e if [ -d zeek-testing ]; then - cd zeek-testing - make + cd zeek-testing + if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then + make + else + make leaks + fi - if [ $? -ne 0 ]; then - showdiag - ret=1 - fi + if [ $? -ne 0 ]; then + showdiag + ret=1 + fi - cd .. + cd .. fi if [ -d zeek-testing-private ]; then - cd zeek-testing-private - make + cd zeek-testing-private + if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then + make + else + make leaks + fi - if [ $? -ne 0 ]; then - showdiag - ret=1 - fi + if [ $? -ne 0 ]; then + showdiag + ret=1 + fi - cd .. + cd .. fi cd ../.. @@ -359,8 +366,6 @@ elif [ "$distro" = "travis" ]; then # a single line when viewing the "Job log" on the Travis CI web site). if [ "$step" = "build" ]; then build - elif [ "$step" = "build_leaktests" ]; then - build_leaktests elif [ "$step" = "run" ]; then run elif [ "$step" = "all" ]; then