mirror of
https://github.com/zeek/zeek.git
synced 2025-10-15 04:58:21 +00:00
Add Cirrus CI config
This commit is contained in:
Jon Siwek
parent
92c2074193
commit
cd9fec7bdb
82 changed files with 692 additions and 149 deletions
|
|
@ -1,429 +0,0 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# This script (along with the .travis.yml file) is used by Travis CI to
|
||||
# build Zeek and run the tests.
|
||||
#
|
||||
# This script can also be used outside of Travis (the "all" build step is
|
||||
# especially convenient in this case). Note that if you use this script
|
||||
# outside of Travis then you will need to fetch the private tests manually
|
||||
# (if you don't, then the private tests will be skipped).
|
||||
|
||||
LEAK_TEST_DISTRO="ubuntu_18.04_leaktest"
|
||||
COVERITY_DISTRO="ubuntu_18.04"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 CMD DISTRO"
|
||||
echo " CMD is a build step:"
|
||||
echo " install: install prereqs"
|
||||
echo " build: build zeek"
|
||||
echo " build_leaktests: build zeek with memory leak testing enabled"
|
||||
echo " run: run the tests"
|
||||
echo " all: do all of the above"
|
||||
echo " DISTRO is a Linux distro, 'travis' to run without docker, or 'coverity' to run a coverity scan"
|
||||
}
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
step=$1
|
||||
distro=$2
|
||||
|
||||
case $step in
|
||||
install) ;;
|
||||
build) ;;
|
||||
run) ;;
|
||||
all) ;;
|
||||
*) echo "Error: unknown build step: $step"; usage; exit 1 ;;
|
||||
esac
|
||||
|
||||
|
||||
# Install the coverity tools.
|
||||
install_coverity() {
|
||||
rm -rf coverity_tool.tgz coverity-tools cov-analysis*
|
||||
|
||||
echo "Downloading coverity tools..."
|
||||
wget -nv https://scan.coverity.com/download/cxx/linux64 --post-data "token=${COV_TOKEN}&project=Bro" -O coverity_tool.tgz
|
||||
tar xzf coverity_tool.tgz
|
||||
rm coverity_tool.tgz
|
||||
mv cov-analysis* coverity-tools
|
||||
}
|
||||
|
||||
|
||||
# Build Zeek with the coverity tools.
|
||||
build_coverity() {
|
||||
# Cleanup any previous build (this is really only necessary if running this
|
||||
# outside of Travis).
|
||||
make distclean > /dev/null
|
||||
|
||||
./configure --prefix=`pwd`/build/root --enable-debug --disable-broker-tests --disable-python --disable-zeekctl
|
||||
|
||||
export PATH=`pwd`/coverity-tools/bin:$PATH
|
||||
cd build
|
||||
cov-build --dir cov-int make -j 4
|
||||
cd ..
|
||||
}
|
||||
|
||||
|
||||
# Create a tar file and send it to coverity.
|
||||
run_coverity() {
|
||||
EMAIL=zeek-commits-internal@zeek.org
|
||||
FILE=myproject.tgz
|
||||
VER=`cat VERSION`
|
||||
DESC=`git rev-parse HEAD`
|
||||
|
||||
cd build
|
||||
echo "Creating tar file and sending to coverity..."
|
||||
tar czf ${FILE} cov-int
|
||||
curl --form token=${COV_TOKEN} --form email=${EMAIL} --form file=@${FILE} --form "version=${VER}" --form "description=${DESC}" https://scan.coverity.com/builds?project=Bro
|
||||
}
|
||||
|
||||
|
||||
# Create a docker container, and install all packages needed to build Zeek.
|
||||
install_in_docker() {
|
||||
local_distro=$distro
|
||||
case $distro in
|
||||
centos_7)
|
||||
distro_cmds="yum -y install gdb make gcc gcc-c++ flex bison libpcap-devel openssl-devel git openssl which centos-release-scl && yum -y install devtoolset-7 && yum -y install epel-release && yum -y install cmake3"
|
||||
;;
|
||||
debian_9)
|
||||
distro_cmds="apt-get update; apt-get -y install gdb cmake make flex bison python libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils clang-7 libc++-7-dev libc++abi-7-dev; update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100; update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100"
|
||||
;;
|
||||
fedora_30)
|
||||
distro_cmds="yum -y install gdb cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel git sqlite findutils which zlib-devel; ln -s /usr/bin/python3 /usr/local/bin/python"
|
||||
;;
|
||||
ubuntu_16.04)
|
||||
distro_cmds="apt-get update; apt-get -y install gdb cmake make flex bison python libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils wget xz-utils; wget -q https://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz; mkdir /clang-9; tar --strip-components=1 -C /clang-9 -xvf clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz; update-alternatives --install /usr/bin/cc cc /clang-9/bin/clang 100; update-alternatives --install /usr/bin/c++ c++ /clang-9/bin/clang++ 100"
|
||||
;;
|
||||
ubuntu_18.04)
|
||||
distro_cmds="apt-get update; apt-get -y install wget xz-utils gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils; ln -s /usr/bin/python3 /usr/local/bin/python"
|
||||
;;
|
||||
${LEAK_TEST_DISTRO})
|
||||
distro_cmds="apt-get update; apt-get -y install gdb cmake make gcc g++ flex bison python3 libpcap-dev libssl-dev zlib1g-dev libkrb5-dev git sqlite3 curl bsdmainutils google-perftools libgoogle-perftools4 libgoogle-perftools-dev; ln -s /usr/bin/python3 /usr/local/bin/python"
|
||||
local_distro="ubuntu_18.04"
|
||||
;;
|
||||
*)
|
||||
echo "Error: distro ${distro} is not recognized by this script"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
docker_image=`echo $local_distro | tr '_' ':'`
|
||||
docker run --name zeektest -id -v "`pwd`:/zeek" -w /zeek ${docker_image} sh
|
||||
docker exec zeektest sh -c "${distro_cmds}"
|
||||
}
|
||||
|
||||
|
||||
# Build Zeek in a docker container.
|
||||
build_in_docker() {
|
||||
recursed_distro=travis
|
||||
|
||||
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
|
||||
recursed_distro=coverity
|
||||
fi
|
||||
|
||||
# Pass the distro as a different environment variable name to docker since
|
||||
# the script will set $distro to "travis" as part of the invocation.
|
||||
docker exec -e COV_TOKEN -e BUILD_DISTRO=${distro} zeektest sh testing/scripts/travis-job build ${recursed_distro}
|
||||
}
|
||||
|
||||
|
||||
# Run Zeek tests in a docker container.
|
||||
run_in_docker() {
|
||||
recursed_distro=travis
|
||||
|
||||
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
|
||||
recursed_distro=coverity
|
||||
fi
|
||||
|
||||
# Pass the distro as a different environment variable name to docker since
|
||||
# the script will set $distro to "travis" as part of the invocation.
|
||||
docker exec -t -e TRAVIS -e TRAVIS_PULL_REQUEST -e TESTING_PRIVATE_DEPLOYKEY -e COV_TOKEN -e BUILD_DISTRO=${distro} zeektest sh testing/scripts/travis-job run ${recursed_distro}
|
||||
}
|
||||
|
||||
update_env() {
|
||||
if [ "${BUILD_DISTRO}" = "centos_7" ]; then
|
||||
source /opt/rh/devtoolset-7/enable
|
||||
elif [ "${BUILD_DISTRO}" = "debian_9" ]; then
|
||||
export CXXFLAGS="-stdlib=libc++"
|
||||
elif [ "${BUILD_DISTRO}" = "ubuntu_16.04" ]; then
|
||||
export CXXFLAGS="-stdlib=libc++"
|
||||
export LD_LIBRARY_PATH=/clang-9/lib
|
||||
fi
|
||||
}
|
||||
|
||||
# Build Zeek.
|
||||
build() {
|
||||
# Cleanup any previous build (this is really only necessary if running this
|
||||
# outside of Travis).
|
||||
make distclean > /dev/null
|
||||
|
||||
update_env
|
||||
|
||||
# Skip building Broker tests, python bindings, and zeekctl, as these are
|
||||
# not needed by the Zeek tests. If the distro is set for leak tests, enable
|
||||
# those options as well.
|
||||
if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
|
||||
./configure --build-type=Release --disable-broker-tests --enable-cpp-tests --disable-python --disable-zeekctl && make -j 2
|
||||
else
|
||||
echo "Configuring zeek to build for leak testing"
|
||||
./configure --build-type=Debug --disable-broker-tests --disable-python --disable-zeekctl --enable-perftools --enable-perftools-debug && make -j 2
|
||||
fi
|
||||
}
|
||||
|
||||
# Get the private tests.
|
||||
get_private_tests() {
|
||||
if [ "${TRAVIS}" != "true" ]; then
|
||||
# When not running in the Travis environment, just skip trying to get
|
||||
# the private tests.
|
||||
echo "Note: skipping private tests (to run them, do a git clone of the private testing repo in the 'testing/external' directory before running this script)."
|
||||
elif [ -n "$TESTING_PRIVATE_DEPLOYKEY" ]; then
|
||||
echo "$TESTING_PRIVATE_DEPLOYKEY" > travis_key.b64
|
||||
base64 --decode travis_key.b64 > travis_key
|
||||
rm travis_key.b64
|
||||
chmod 600 travis_key
|
||||
mkdir -p ~/.ssh
|
||||
mv travis_key ~/.ssh/id_rsa
|
||||
echo "Host *" >> ~/.ssh/config
|
||||
echo " StrictHostKeyChecking no" >> ~/.ssh/config
|
||||
chmod 400 ~/.ssh/config
|
||||
git clone git@github.com:zeek/zeek-testing-private
|
||||
rm ~/.ssh/id_rsa
|
||||
elif [ -n "${TRAVIS_PULL_REQUEST}" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then
|
||||
# For pull request builds, the private key is not available, so skip
|
||||
# the private tests to avoid failing.
|
||||
echo "Note: skipping private tests because encrypted env. variables are not available in pull request builds."
|
||||
else
|
||||
echo "Error: cannot get private tests because encrypted env. variables are not defined."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# Run Zeek tests.
|
||||
run() {
|
||||
update_env
|
||||
|
||||
ulimit -c unlimited
|
||||
ulimit -a
|
||||
ret=0
|
||||
|
||||
if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
|
||||
echo
|
||||
echo "Running unit tests ##################################################"
|
||||
echo
|
||||
|
||||
set +e
|
||||
( cd build && . ./zeek-path-dev.sh && zeek --test )
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
ret=1
|
||||
fi
|
||||
|
||||
set -e
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Running baseline tests ##############################################"
|
||||
echo
|
||||
cd testing/btest
|
||||
|
||||
set +e
|
||||
# Must specify a value for "-j" option, otherwise Travis uses a huge value.
|
||||
if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
|
||||
../../aux/btest/btest -j 4 -d
|
||||
else
|
||||
../../aux/btest/btest -j 4 -d -g leaks
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
ret=1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
echo
|
||||
echo "Getting external tests ##############################################"
|
||||
echo
|
||||
cd ../external
|
||||
|
||||
if [ ! -d zeek-testing ]; then
|
||||
make init
|
||||
fi
|
||||
|
||||
if [ -d zeek-testing ]; then
|
||||
commit=`cat commit-hash.zeek-testing`
|
||||
echo "Checking out $commit"
|
||||
( cd zeek-testing && git checkout -q $commit )
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
if [ ! -d zeek-testing-private ]; then
|
||||
get_private_tests
|
||||
fi
|
||||
|
||||
if [ -d zeek-testing-private ]; then
|
||||
commit=`cat commit-hash.zeek-testing-private`
|
||||
echo "Checking out $commit"
|
||||
( cd zeek-testing-private && git checkout -q $commit )
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Running external tests ##############################################"
|
||||
echo
|
||||
|
||||
set +e
|
||||
|
||||
if [ -d zeek-testing ]; then
|
||||
cd zeek-testing
|
||||
if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
|
||||
make
|
||||
else
|
||||
make leaks
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
showdiag
|
||||
ret=1
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
|
||||
if [ -d zeek-testing-private ]; then
|
||||
cd zeek-testing-private
|
||||
if [ "${BUILD_DISTRO}" != "${LEAK_TEST_DISTRO}" ]; then
|
||||
make
|
||||
else
|
||||
make leaks
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
showdiag
|
||||
ret=1
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
|
||||
cd ../..
|
||||
|
||||
echo
|
||||
echo "Result code after running tests: $ret"
|
||||
|
||||
if [ $ret -ne 0 ]; then
|
||||
COREFILES=`find testing/btest/.tmp testing/external/*/.tmp -type f -name core*`
|
||||
|
||||
echo
|
||||
echo "Search for core dumps ##############################################"
|
||||
echo
|
||||
echo $COREFILES
|
||||
|
||||
for cf in $COREFILES; do
|
||||
echo
|
||||
echo "############# Begin stack trace for $cf ###############"
|
||||
gdb build/src/zeek -c "$cf" -ex "thread apply all bt" -ex "set pagination 0" -batch;
|
||||
echo "############# End stack trace for $cf #################"
|
||||
echo
|
||||
done
|
||||
fi
|
||||
|
||||
# If we get here, then external tests were successful.
|
||||
exit $ret
|
||||
}
|
||||
|
||||
# Show failed tests (not skipped tests) from diag.log when a test fails.
|
||||
showdiag() {
|
||||
f=diag.log
|
||||
|
||||
grep -qs '... failed$' $f && \
|
||||
echo && \
|
||||
echo "Output of failed external tests #####################################" && \
|
||||
echo && \
|
||||
grep -v "... not available, skipped" $f
|
||||
}
|
||||
|
||||
# Remove the docker container.
|
||||
remove_container() {
|
||||
echo "Removing the docker container..."
|
||||
docker rm -f zeektest > /dev/null
|
||||
}
|
||||
|
||||
|
||||
if [ ! -f testing/scripts/travis-job ]; then
|
||||
echo "Error: must change directory to root of zeek source tree before running this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [ "$distro" = "coverity" ]; then
|
||||
# Check if the project token is available (this is a secret value and
|
||||
# should not be hard-coded in this script). This value can be found by
|
||||
# logging into the coverity scan web site and looking in the project
|
||||
# settings.
|
||||
if [ -z "${COV_TOKEN}" ]; then
|
||||
echo "Error: COV_TOKEN is not defined (should be defined in environment variables section of Travis settings for this repo)"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then
|
||||
# This is a Travis CI cron job which only runs the Coverity build inside
|
||||
# a single container.
|
||||
|
||||
if [ "$distro" != "${COVERITY_DISTRO}" ]; then
|
||||
echo "Coverity scan is performed on ${COVERITY_DISTRO}"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$distro" = "coverity" ]; then
|
||||
# The "build" and "run" steps are split up into separate steps because the
|
||||
# build outputs thousands of lines (which are conveniently collapsed into
|
||||
# a single line when viewing the "Job log" on the Travis CI web site).
|
||||
if [ "$step" = "install" ]; then
|
||||
echo "No 'install' step needed for Coverity build"
|
||||
elif [ "$step" = "build" ]; then
|
||||
install_coverity
|
||||
build_coverity
|
||||
elif [ "$step" = "run" ]; then
|
||||
run_coverity
|
||||
elif [ "$step" = "all" ]; then
|
||||
install_coverity
|
||||
build_coverity
|
||||
run_coverity
|
||||
fi
|
||||
elif [ "$distro" = "travis" ]; then
|
||||
# Build Zeek and run tests.
|
||||
|
||||
# The "build" and "run" steps are split up into separate steps because the
|
||||
# build outputs thousands of lines (which are conveniently collapsed into
|
||||
# a single line when viewing the "Job log" on the Travis CI web site).
|
||||
if [ "$step" = "build" ]; then
|
||||
build
|
||||
elif [ "$step" = "run" ]; then
|
||||
run
|
||||
elif [ "$step" = "all" ]; then
|
||||
build
|
||||
run
|
||||
fi
|
||||
else
|
||||
# Build Zeek and run tests in a docker container.
|
||||
|
||||
if [ "$step" = "install" ]; then
|
||||
install_in_docker
|
||||
elif [ "$step" = "build" ]; then
|
||||
build_in_docker
|
||||
elif [ "$step" = "run" ]; then
|
||||
run_in_docker
|
||||
elif [ "$step" = "all" ]; then
|
||||
install_in_docker
|
||||
build_in_docker
|
||||
run_in_docker
|
||||
# If all tests pass, then remove the docker container.
|
||||
remove_container
|
||||
fi
|
||||
fi
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#! /usr/bin/env bash
|
||||
|
||||
# Sleeps until a process id no longer exists.
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
>&2 echo "usage: $0 <pid to wait for> <max secs to wait>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wait_pid=$1
|
||||
max_wait=$2
|
||||
wait_count=0
|
||||
|
||||
while kill -0 $wait_pid &> /dev/null; do
|
||||
let "wait_count += 1"
|
||||
|
||||
if [[ $wait_count -ge $max_wait ]]; then
|
||||
>&2 echo "error: process $wait_pid still exists after $max_wait seconds"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue