Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improve the travis-job script to work outside of Travis

Browse source 
Improved the travis-job test script so that it works outside of Travis.
This can be useful to test changes to the docker container config before
pushing to master, for testing changes on a branch, or for debugging
some problem with the tests running on Travis.  This required the
following changes:
-The script no longer requires the TRAVIS env. variable to be set.
-Added an "all" command as a more convenient way to run all steps in a
 build.
-Added a "coverity" command-line option to do a coverity build.
-Before building Bro or downloading coverity tools, do a cleanup from any
 previous build.

Also other minor improvements (code comments, reorganization, etc.).
This commit is contained in:
Daniel Thayer 2018-08-09 13:58:43 -05:00
parent 29c179c30d
commit 5d3ef4daf4
1 changed files with 140 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

200
testing/scripts/travis-job
View file

@ -2,58 +2,82 @@
#
# This script (along with the .travis.yml file) is used by Travis CI to
# build Bro and run the tests.
#
# This script can also be used outside of Travis (the "all" build step is
# especially convenient in this case). Note that if you use this script
# outside of Travis then you will need to fetch the private tests manually
# (if you don't, then the private tests will be skipped).
usage() {
echo "usage: $0 CMD DISTRO"
echo " CMD is a build step:"
echo " install: install prereqs"
echo " build: build bro"
echo " run: run the tests"
echo " all: do all of the above"
echo " DISTRO is a Linux distro, 'travis' to run without docker, or 'coverity' to run a coverity scan"
}
if [ $# -ne 2 ]; then
echo "usage: $0 CMD DISTRO"
echo " CMD is a build step (install, build, or run)"
echo " DISTRO is a Linux distro, or 'travis' to run in Travis without docker"
usage
exit 1
fi
step=$1
distro=$2
# Build Bro with the coverity tools.
build_coverity() {
# Get the coverity tools
set -e
case $step in
install) ;;
build) ;;
run) ;;
all) ;;
*) echo "Error: unknown build step: $step"; usage; exit 1 ;;
esac
if [ -z "${COV_TOKEN}" ]; then
echo "Error: COV_TOKEN is not defined (should be defined in environment variables section of Travis settings for this repo)"
exit 1
fi
# Install the coverity tools.
install_coverity() {
rm -rf coverity_tool.tgz coverity-tools cov-analysis*
echo "Downloading coverity tools..."
wget -nv https://scan.coverity.com/download/cxx/linux64 --post-data "token=${COV_TOKEN}&project=Bro" -O coverity_tool.tgz
tar xzf coverity_tool.tgz
mv cov-analysis* coverity-tools
rm coverity_tool.tgz
mv cov-analysis* coverity-tools
}
# Configure Bro
./configure --prefix=`pwd`/build/root --enable-debug --disable-perftools
# Build Bro with coverity tools
# Build Bro with the coverity tools.
build_coverity() {
# Cleanup any previous build (this is really only necessary if running this
# outside of Travis).
make distclean > /dev/null
./configure --prefix=`pwd`/build/root --enable-debug --disable-perftools --disable-broker-tests --disable-python --disable-broctl
export PATH=`pwd`/coverity-tools/bin:$PATH
cd build
cov-build --dir cov-int make -j 4
cd ..
}
# Create a tar file and send it to coverity.
run_coverity() {
set -e
EMAIL=bro-commits-internal@bro.org
FILE=myproject.bz2
FILE=myproject.tgz
VER=`cat VERSION`
DESC=`git rev-parse HEAD`
cd build
tar cjf ${FILE} cov-int
echo "Creating tar file and sending to coverity..."
tar czf ${FILE} cov-int
curl --form token=${COV_TOKEN} --form email=${EMAIL} --form file=@${FILE} --form "version=${VER}" --form "description=${DESC}" https://scan.coverity.com/builds?project=Bro
}
# Setup a docker container.
setup_docker() {
# Create a docker container, and install all packages needed to build Bro.
install_in_docker() {
case $distro in
centos_7)
distro_cmds="yum -y install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel git openssl which"
@ -83,20 +107,24 @@ setup_docker() {
# Build bro in a docker container.
build_docker() {
docker exec -e TRAVIS brotest sh testing/scripts/travis-job $step travis
build_in_docker() {
docker exec brotest sh testing/scripts/travis-job build travis
}
# Run Bro tests in a docker container.
run_docker() {
run_in_docker() {
prepare_env
docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e trav_key -e trav_iv brotest sh testing/scripts/travis-job $step travis
docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e trav_key -e trav_iv brotest sh testing/scripts/travis-job run travis
}
# Build Bro.
build() {
# Cleanup any previous build (this is really only necessary if running this
# outside of Travis).
make distclean > /dev/null
# Skip building broker tests, python bindings, and broctl, as these are
# not needed by the bro tests.
./configure --build-type=Release --disable-broker-tests --disable-python --disable-broctl && make -j 2
@ -107,6 +135,9 @@ build() {
# hard-coded multiple times in this script.
prepare_env() {
if [ -z "$trav_key" ]; then
# This hash value is found by logging into the Travis CI website,
# and looking at the settings in the bro repo (look in the
# "Environment Variables" section).
hash=6a6fe747ff7b
eval "trav_key=\$encrypted_${hash}_key"
eval "trav_iv=\$encrypted_${hash}_iv"
@ -117,27 +148,15 @@ prepare_env() {
}
# Run Bro tests.
run() {
echo
echo "Running unit tests ##################################################"
echo
cd testing/btest
# Must specify a value for "-j" option, otherwise Travis uses a huge value.
../../aux/btest/btest -j 4 -d
ret=$?
echo
echo "Getting external tests ##############################################"
echo
cd ../external
set -e
make init
# Get the private tests.
get_private_tests() {
prepare_env
if [ -n "$trav_key" ] && [ -n "$trav_iv" ]; then
if [ "${TRAVIS}" != "true" ]; then
# When not running in the Travis environment, just skip trying to get
# the private tests.
echo "Note: skipping private tests (to run them, do a git clone of the private testing repo in the 'testing/external' directory before running this script)."
elif [ -n "$trav_key" ] && [ -n "$trav_iv" ]; then
curl https://www.bro.org/static/travis-ci/travis_key.enc -o travis_key.enc
openssl aes-256-cbc -K $trav_key -iv $trav_iv -in travis_key.enc -out travis_key -d
chmod 600 travis_key
@ -154,6 +173,34 @@ run() {
echo "Error: cannot get private tests because encrypted env. variables are not defined."
exit 1
fi
}
# Run Bro tests.
run() {
echo
echo "Running unit tests ##################################################"
echo
cd testing/btest
set +e
# Must specify a value for "-j" option, otherwise Travis uses a huge value.
../../aux/btest/btest -j 4 -d
ret=$?
set -e
echo
echo "Getting external tests ##############################################"
echo
cd ../external
if [ ! -d bro-testing ]; then
make init
fi
if [ ! -d bro-testing-private ]; then
get_private_tests
fi
echo
echo "Running external tests ##############################################"
@ -164,9 +211,8 @@ run() {
exit $ret
}
# Output the contents of diag.log when a test fails.
# Show failed tests (not skipped tests) from diag.log when a test fails.
showdiag() {
# Show failed tests only, not skipped tests.
f=bro-testing/diag.log
grep -qs '... failed$' $f && \
@ -178,18 +224,22 @@ showdiag() {
exit 1
}
if [ "$step" != "install" ] && [ "$step" != "build" ] && [ "$step" != "run" ]; then
echo "Error: unknown build step: $step"
# Remove the docker container.
remove_container() {
echo "Removing the docker container..."
docker rm -f brotest > /dev/null
}
if [ ! -f testing/scripts/travis-job ]; then
echo "Error: must change directory to root of bro source tree before running this script."
exit 1
fi
if [ "${TRAVIS}" != "true" ]; then
echo "$0: this script is intended for Travis CI"
exit 1
fi
set -e
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
# Run the coverity scan from a Travis CI cron job.
# This is a Travis CI cron job, so check the job number.
# Extract second component of the job number.
if [ -z "${TRAVIS_JOB_NUMBER}" ]; then
@ -204,14 +254,35 @@ if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
echo "Coverity scan is performed only in the first job of this build"
exit 0
fi
fi
# This is split up into two steps because the build outputs thousands of
# lines (which are conveniently collapsed into a single line in the
# "Job log" on the Travis CI web site).
if [ "$step" = "build" ]; then
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [ "$distro" = "coverity" ]; then
# Run coverity scan when this script is run from a Travis cron job, or
# if the user specifies the "coverity" distro.
# Check if the project token is available (this is a secret value and
# should not be hard-coded in this script). This value can be found by
# logging into the coverity scan web site and looking in the project
# settings.
if [ -z "${COV_TOKEN}" ]; then
echo "Error: COV_TOKEN is not defined (should be defined in environment variables section of Travis settings for this repo)"
exit 1
fi
# The "build" and "run" steps are split up into separate steps because the
# build outputs thousands of lines (which are conveniently collapsed into
# a single line when viewing the "Job log" on the Travis CI web site).
if [ "$step" = "install" ]; then
install_coverity
elif [ "$step" = "build" ]; then
build_coverity
elif [ "$step" = "run" ]; then
run_coverity
elif [ "$step" = "all" ]; then
install_coverity
build_coverity
run_coverity
fi
elif [ "$distro" = "travis" ]; then
# Build bro and run tests.
@ -223,15 +294,24 @@ elif [ "$distro" = "travis" ]; then
build
elif [ "$step" = "run" ]; then
run
elif [ "$step" = "all" ]; then
build
run
fi
else
# Build bro and run tests in a docker container.
if [ "$step" = "install" ]; then
setup_docker
install_in_docker
elif [ "$step" = "build" ]; then
build_docker
build_in_docker
elif [ "$step" = "run" ]; then
run_docker
run_in_docker
elif [ "$step" = "all" ]; then
install_in_docker
build_in_docker
run_in_docker
# If all tests pass, then remove the docker container.
remove_container
fi
fi