mirror of
https://github.com/zeek/zeek.git
synced 2025-10-14 04:28:20 +00:00
38505589e1
Using AddressSanitizer/LeakSanitizer is better. It covers the full unit/baseline test suite by default without requiring one to write specific memory leak tests. It also covers other types of memory errors besides just leaks.
415 lines
13 KiB
Bash
Executable file
415 lines
13 KiB
Bash
Executable file
#!/bin/sh
|
|
#
|
|
# This script (along with the .travis.yml file) is used by Travis CI to
|
|
# build Zeek and run the tests.
|
|
#
|
|
# This script can also be used outside of Travis (the "all" build step is
|
|
# especially convenient in this case). Note that if you use this script
|
|
# outside of Travis then you will need to fetch the private tests manually
|
|
# (if you don't, then the private tests will be skipped).
|
|
|
|
LEAK_TEST_DISTRO="ubuntu_18.04_leaktest"
|
|
COVERITY_DISTRO="ubuntu_18.04"
|
|
|
|
usage() {
|
|
echo "usage: $0 CMD DISTRO"
|
|
echo " CMD is a build step:"
|
|
echo " install: install prereqs"
|
|
echo " build: build zeek"
|
|
echo " build_leaktests: build zeek with memory leak testing enabled"
|
|
echo " run: run the tests"
|
|
echo " all: do all of the above"
|
|
echo " DISTRO is a Linux distro, 'travis' to run without docker, or 'coverity' to run a coverity scan"
|
|
}
|
|
|
|
if [ $# -ne 2 ]; then
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
step=$1
|
|
distro=$2
|
|
|
|
case $step in
|
|
install) ;;
|
|
build) ;;
|
|
run) ;;
|
|
all) ;;
|
|
*) echo "Error: unknown build step: $step"; usage; exit 1 ;;
|
|
esac
|
|
|
|
|
|
# Install the coverity tools.
|
|
install_coverity() {
|
|
rm -rf coverity_tool.tgz coverity-tools cov-analysis*
|
|
|
|
echo "Downloading coverity tools..."
|
|
wget -nv https://scan.coverity.com/download/cxx/linux64 --post-data "token=${COV_TOKEN}&project=Bro" -O coverity_tool.tgz
|
|
tar xzf coverity_tool.tgz
|
|
rm coverity_tool.tgz
|
|
mv cov-analysis* coverity-tools
|
|
}
|
|
|
|
|
|
# Build Zeek with the coverity tools.
|
|
build_coverity() {
|
|
# Cleanup any previous build (this is really only necessary if running this
|
|
# outside of Travis).
|
|
make distclean > /dev/null
|
|
|
|
./configure --prefix=`pwd`/build/root --enable-debug --disable-broker-tests --disable-python --disable-zeekctl
|
|
|
|
export PATH=`pwd`/coverity-tools/bin:$PATH
|
|
cd build
|
|
cov-build --dir cov-int make -j 4
|
|
cd ..
|
|
}
|
|
|
|
|
|
# Create a tar file and send it to coverity.
|
|
run_coverity() {
|
|
EMAIL=zeek-commits-internal@zeek.org
|
|
FILE=myproject.tgz
|
|
VER=`cat VERSION`
|
|
DESC=`git rev-parse HEAD`
|
|
|
|
cd build
|
|
echo "Creating tar file and sending to coverity..."
|
|
tar czf ${FILE} cov-int
|
|
curl --form token=${COV_TOKEN} --form email=${EMAIL} --form file=@${FILE} --form "version=${VER}" --form "description=${DESC}" https://scan.coverity.com/builds?project=Bro
|
|
}
|
|
|
|
|
|
# Create a docker container, and install all packages needed to build Zeek.
|
|
install_in_docker() {
|
|
local_distro=$distro
|
|
case $distro in
|
|
centos_7)
|
|
distro_cmds="yum -y install gdb make gcc gcc-c++ flex bison libpcap-devel openssl-devel git openssl which centos-release-scl && yum -y install devtoolset-7 && yum -y install epel-release && yum -y install cmake3"
|
|
;;
|
|
debian_9)
|
|
distro_cmds="apt-get update; apt-get -y install gdb cmake make flex bison python libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils clang-7 libc++-7-dev libc++abi-7-dev; update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100; update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100"
|
|
;;
|
|
fedora_30)
|
|
distro_cmds="yum -y install gdb cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel git sqlite findutils which zlib-devel; ln -s /usr/bin/python3 /usr/local/bin/python"
|
|
;;
|
|
ubuntu_16.04)
|
|
distro_cmds="apt-get update; apt-get -y install gdb cmake make flex bison python libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils wget xz-utils; wget -q https://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz; mkdir /clang-9; tar --strip-components=1 -C /clang-9 -xvf clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz; update-alternatives --install /usr/bin/cc cc /clang-9/bin/clang 100; update-alternatives --install /usr/bin/c++ c++ /clang-9/bin/clang++ 100"
|
|
;;
|
|
ubuntu_18.04)
|
|
distro_cmds="apt-get update; apt-get -y install wget xz-utils gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils; ln -s /usr/bin/python3 /usr/local/bin/python"
|
|
;;
|
|
${LEAK_TEST_DISTRO})
|
|
distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils; ln -s /usr/bin/python3 /usr/local/bin/python"
|
|
local_distro="ubuntu_18.04"
|
|
;;
|
|
*)
|
|
echo "Error: distro ${distro} is not recognized by this script"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
docker_image=`echo $local_distro | tr '_' ':'`
|
|
docker run --name zeektest -id -v "`pwd`:/zeek" -w /zeek ${docker_image} sh
|
|
docker exec zeektest sh -c "${distro_cmds}"
|
|
}
|
|
|
|
|
|
# Build Zeek in a docker container.
|
|
build_in_docker() {
|
|
recursed_distro=travis
|
|
|
|
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
|
|
recursed_distro=coverity
|
|
fi
|
|
|
|
# Pass the distro as a different environment variable name to docker since
|
|
# the script will set $distro to "travis" as part of the invocation.
|
|
docker exec -e COV_TOKEN -e BUILD_DISTRO=${distro} zeektest sh ci/travis-job build ${recursed_distro}
|
|
}
|
|
|
|
|
|
# Run Zeek tests in a docker container.
|
|
run_in_docker() {
|
|
recursed_distro=travis
|
|
|
|
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
|
|
recursed_distro=coverity
|
|
fi
|
|
|
|
# Pass the distro as a different environment variable name to docker since
|
|
# the script will set $distro to "travis" as part of the invocation.
|
|
docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e TESTING_PRIVATE_DEPLOYKEY -e COV_TOKEN -e BUILD_DISTRO=${distro} zeektest sh ci/travis-job run ${recursed_distro}
|
|
}
|
|
|
|
update_env() {
|
|
if [ "${BUILD_DISTRO}" = "centos_7" ]; then
|
|
source /opt/rh/devtoolset-7/enable
|
|
elif [ "${BUILD_DISTRO}" = "debian_9" ]; then
|
|
export CXXFLAGS="-stdlib=libc++"
|
|
elif [ "${BUILD_DISTRO}" = "ubuntu_16.04" ]; then
|
|
export CXXFLAGS="-stdlib=libc++"
|
|
export LD_LIBRARY_PATH=/clang-9/lib
|
|
fi
|
|
}
|
|
|
|
# Build Zeek.
|
|
build() {
|
|
# Cleanup any previous build (this is really only necessary if running this
|
|
# outside of Travis).
|
|
make distclean > /dev/null
|
|
|
|
update_env
|
|
|
|
# Skip building Broker tests, python bindings, and zeekctl, as these are
|
|
# not needed by the Zeek tests. If the distro is set for leak tests, enable
|
|
# those options as well.
|
|
if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
|
|
./configure --build-type=Release --disable-broker-tests --enable-cpp-tests --disable-python --disable-zeekctl && make -j 2
|
|
else
|
|
echo "Configuring zeek to build for leak testing"
|
|
./configure --build-type=Debug --disable-broker-tests --enable-cpp-tests --disable-python --disable-zeekctl --sanitizers=address && make -j 2
|
|
fi
|
|
}
|
|
|
|
# Get the private tests.
|
|
get_private_tests() {
|
|
if [ "${TRAVIS}" != "true" ]; then
|
|
# When not running in the Travis environment, just skip trying to get
|
|
# the private tests.
|
|
echo "Note: skipping private tests (to run them, do a git clone of the private testing repo in the 'testing/external' directory before running this script)."
|
|
elif [ -n "$TESTING_PRIVATE_DEPLOYKEY" ]; then
|
|
echo "$TESTING_PRIVATE_DEPLOYKEY" > travis_key.b64
|
|
base64 --decode travis_key.b64 > travis_key
|
|
rm travis_key.b64
|
|
chmod 600 travis_key
|
|
mkdir -p ~/.ssh
|
|
mv travis_key ~/.ssh/id_rsa
|
|
echo "Host *" >> ~/.ssh/config
|
|
echo " StrictHostKeyChecking no" >> ~/.ssh/config
|
|
chmod 400 ~/.ssh/config
|
|
git clone git@github.com:zeek/zeek-testing-private
|
|
rm ~/.ssh/id_rsa
|
|
elif [ -n "${TRAVIS_PULL_REQUEST}" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then
|
|
# For pull request builds, the private key is not available, so skip
|
|
# the private tests to avoid failing.
|
|
echo "Note: skipping private tests because encrypted env. variables are not available in pull request builds."
|
|
else
|
|
echo "Error: cannot get private tests because encrypted env. variables are not defined."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
|
|
# Run Zeek tests.
|
|
run() {
|
|
update_env
|
|
|
|
ulimit -c unlimited
|
|
ulimit -a
|
|
ret=0
|
|
|
|
echo
|
|
echo "Running unit tests ##################################################"
|
|
echo
|
|
|
|
set +e
|
|
( cd build && . ./zeek-path-dev.sh && zeek --test )
|
|
|
|
if [ $? -ne 0 ]; then
|
|
ret=1
|
|
fi
|
|
|
|
set -e
|
|
|
|
echo
|
|
echo "Running baseline tests ##############################################"
|
|
echo
|
|
cd testing/btest
|
|
|
|
set +e
|
|
# Must specify a value for "-j" option, otherwise Travis uses a huge value.
|
|
../../aux/btest/btest -j 4 -d
|
|
|
|
if [ $? -ne 0 ]; then
|
|
ret=1
|
|
fi
|
|
|
|
set -e
|
|
|
|
echo
|
|
echo "Getting external tests ##############################################"
|
|
echo
|
|
cd ../external
|
|
|
|
if [ ! -d zeek-testing ]; then
|
|
make init
|
|
fi
|
|
|
|
if [ -d zeek-testing ]; then
|
|
commit=`cat commit-hash.zeek-testing`
|
|
echo "Checking out $commit"
|
|
( cd zeek-testing && git checkout -q $commit )
|
|
fi
|
|
|
|
echo
|
|
|
|
if [ ! -d zeek-testing-private ]; then
|
|
get_private_tests
|
|
fi
|
|
|
|
if [ -d zeek-testing-private ]; then
|
|
commit=`cat commit-hash.zeek-testing-private`
|
|
echo "Checking out $commit"
|
|
( cd zeek-testing-private && git checkout -q $commit )
|
|
fi
|
|
|
|
echo
|
|
echo "Running external tests ##############################################"
|
|
echo
|
|
|
|
set +e
|
|
|
|
if [ -d zeek-testing ]; then
|
|
cd zeek-testing
|
|
make
|
|
|
|
if [ $? -ne 0 ]; then
|
|
showdiag
|
|
ret=1
|
|
fi
|
|
|
|
cd ..
|
|
fi
|
|
|
|
if [ -d zeek-testing-private ]; then
|
|
cd zeek-testing-private
|
|
make
|
|
|
|
if [ $? -ne 0 ]; then
|
|
showdiag
|
|
ret=1
|
|
fi
|
|
|
|
cd ..
|
|
fi
|
|
|
|
cd ../..
|
|
|
|
echo
|
|
echo "Result code after running tests: $ret"
|
|
|
|
if [ $ret -ne 0 ]; then
|
|
COREFILES=`find testing/btest/.tmp testing/external/*/.tmp -type f -name core*`
|
|
|
|
echo
|
|
echo "Search for core dumps ##############################################"
|
|
echo
|
|
echo $COREFILES
|
|
|
|
for cf in $COREFILES; do
|
|
echo
|
|
echo "############# Begin stack trace for $cf ###############"
|
|
gdb build/src/zeek -c "$cf" -ex "thread apply all bt" -ex "set pagination 0" -batch;
|
|
echo "############# End stack trace for $cf #################"
|
|
echo
|
|
done
|
|
fi
|
|
|
|
# If we get here, then external tests were successful.
|
|
exit $ret
|
|
}
|
|
|
|
# Show failed tests (not skipped tests) from diag.log when a test fails.
|
|
showdiag() {
|
|
f=diag.log
|
|
|
|
grep -qs '... failed$' $f && \
|
|
echo && \
|
|
echo "Output of failed external tests #####################################" && \
|
|
echo && \
|
|
grep -v "... not available, skipped" $f
|
|
}
|
|
|
|
# Remove the docker container.
|
|
remove_container() {
|
|
echo "Removing the docker container..."
|
|
docker rm -f zeektest > /dev/null
|
|
}
|
|
|
|
|
|
if [ ! -f ci/travis-job ]; then
|
|
echo "Error: must change directory to root of zeek source tree before running this script."
|
|
exit 1
|
|
fi
|
|
|
|
set -e
|
|
|
|
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [ "$distro" = "coverity" ]; then
|
|
# Check if the project token is available (this is a secret value and
|
|
# should not be hard-coded in this script). This value can be found by
|
|
# logging into the coverity scan web site and looking in the project
|
|
# settings.
|
|
if [ -z "${COV_TOKEN}" ]; then
|
|
echo "Error: COV_TOKEN is not defined (should be defined in environment variables section of Travis settings for this repo)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
|
|
# This is a Travis CI cron job which only runs the Coverity build inside
|
|
# a single container.
|
|
|
|
if [ "$distro" != "${COVERITY_DISTRO}" ]; then
|
|
echo "Coverity scan is performed on ${COVERITY_DISTRO}"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
if [ "$distro" = "coverity" ]; then
|
|
# The "build" and "run" steps are split up into separate steps because the
|
|
# build outputs thousands of lines (which are conveniently collapsed into
|
|
# a single line when viewing the "Job log" on the Travis CI web site).
|
|
if [ "$step" = "install" ]; then
|
|
echo "No 'install' step needed for Coverity build"
|
|
elif [ "$step" = "build" ]; then
|
|
install_coverity
|
|
build_coverity
|
|
elif [ "$step" = "run" ]; then
|
|
run_coverity
|
|
elif [ "$step" = "all" ]; then
|
|
install_coverity
|
|
build_coverity
|
|
run_coverity
|
|
fi
|
|
elif [ "$distro" = "travis" ]; then
|
|
# Build Zeek and run tests.
|
|
|
|
# The "build" and "run" steps are split up into separate steps because the
|
|
# build outputs thousands of lines (which are conveniently collapsed into
|
|
# a single line when viewing the "Job log" on the Travis CI web site).
|
|
if [ "$step" = "build" ]; then
|
|
build
|
|
elif [ "$step" = "run" ]; then
|
|
run
|
|
elif [ "$step" = "all" ]; then
|
|
build
|
|
run
|
|
fi
|
|
else
|
|
# Build Zeek and run tests in a docker container.
|
|
|
|
if [ "$step" = "install" ]; then
|
|
install_in_docker
|
|
elif [ "$step" = "build" ]; then
|
|
build_in_docker
|
|
elif [ "$step" = "run" ]; then
|
|
run_in_docker
|
|
elif [ "$step" = "all" ]; then
|
|
install_in_docker
|
|
build_in_docker
|
|
run_in_docker
|
|
# If all tests pass, then remove the docker container.
|
|
remove_container
|
|
fi
|
|
fi
|