Merge remote-tracking branch 'origin/topic/dnthayer/travis-ci-updates'

I've moved .travis.job to testing/scripts/travis-job and adapted
.travis.yml accordingly.

* origin/topic/dnthayer/travis-ci-updates:
  Fix information leak in the update-traces script
  Add coverity scan and private testing to Travis CI

.travis.yml

@@ -4,6 +4,7 @@ compiler:
   - gcc
 
 addons:
+  ssh_known_hosts: git.bro.org
   apt:
     sources:
       - ubuntu-toolchain-r-test
@@ -26,14 +27,8 @@ notifications:
     recipients:
     - bro-commits-internal@bro.org
 
-install: ./configure && make -j 4
+before_script: sh testing/scripts/travis-job build
 
-script:
-  - make -C testing/btest btest-verbose
-  - make -C testing/external init && make -C testing/external
+script: sh testing/scripts/travis-job run
 
-after_failure:
-  # Output each diag.log that contains failed test results.
-  - for i in testing/btest/diag.log testing/external/bro-testing/diag.log; do
-    grep -qs '... failed$' $i && cat $i ;
-    done
+after_failure: sh testing/scripts/travis-job failure

testing/external/scripts/update-traces

@@ -2,7 +2,7 @@
 #
 # Downloads all traces as specified in <cwd>/traces.cfg to directory $1.
 #
-# traces.cfg must consist of lines of the form "<url> <md5sum>"
+# traces.cfg must consist of lines of the form "<url> [<http-user>[:<http-password>]]"
 
 if [ "$1" == "" ]; then
     echo "usage: `basename $0` <traces-directory>"
@@ -45,11 +45,15 @@ cat $cfg | while read line; do
 
     if [ "$auth" != "" ]; then
         auth="-u $auth"
+        # Hide the hostname and directory names in output messages
+        safe_url=`echo $url | sed 's#/[A-Za-z].*/#/[hidden]/#'`
+    else
+        safe_url=$url
     fi
 
     # Get the fingerprint file.
     if ! eval "$proxy curl $auth -fsS --anyauth $url.md5sum -o $fp.tmp"; then
-        echo "Error: Could not get $url.md5sum, skipping download."
+        echo "Error: Could not get $safe_url.md5sum, skipping download."
         continue
     fi
 
@@ -64,7 +68,7 @@ cat $cfg | while read line; do
     fi
 
     if [ "$download" = "1" ]; then
-        echo Getting $url ...
+        echo Getting $safe_url ...
         echo
         eval "$proxy curl $auth -f --anyauth $url -o $file"
         echo

testing/scripts/travis-job

@@ -0,0 +1,110 @@
+#!/bin/sh
+
+if [ "${TRAVIS}" != "true" ]; then
+    echo "$0: this script is intended for Travis CI"
+    exit 1
+fi
+
+if [ $# -ne 1 ]; then
+    echo "usage: $0 build|run|failure"
+    exit 1
+fi
+
+step=$1
+
+build() {
+    ./configure && make -j 4
+}
+
+build_coverity() {
+    # Get the coverity tools
+    set -e
+    wget -nv https://scan.coverity.com/download/cxx/linux64 --post-data "token=${COV_TOKEN}&project=Bro" -O coverity_tool.tgz
+    tar xzf coverity_tool.tgz
+    mv cov-analysis* coverity-tools
+    rm coverity_tool.tgz
+
+    # Configure Bro
+    ./configure --prefix=`pwd`/build/root --enable-debug --disable-perftools
+
+    # Build Bro with coverity tools
+    export PATH=`pwd`/coverity-tools/bin:$PATH
+    cd build
+    cov-build --dir cov-int make -j 4
+}
+
+run_coverity() {
+    set -e
+
+    EMAIL=bro-commits-internal@bro.org
+    FILE=myproject.bz2
+    VER=`cat VERSION`
+    DESC=`git rev-parse HEAD`
+
+    cd build
+    tar cjf ${FILE} cov-int
+    curl --form token=${COV_TOKEN} --form email=${EMAIL} --form file=@${FILE} --form version=${VER} --form description=${DESC} https://scan.coverity.com/builds?project=Bro
+}
+
+run() {
+    set -e
+
+    # Run the tests
+    make -C testing/btest btest-verbose
+
+    # Get the test repo
+    make -C testing/external init
+
+    # Get the private test repo
+    curl https://www.bro.org/static/travis-ci/travis_key.enc -o travis_key.enc
+    openssl aes-256-cbc -K $encrypted_6a6fe747ff7b_key -iv $encrypted_6a6fe747ff7b_iv -in travis_key.enc -out travis_key -d
+    chmod 600 travis_key
+    mv travis_key $HOME/.ssh/id_rsa
+    cd testing/external
+    git clone ssh://git@git.bro.org/bro-testing-private
+    cd ../..
+    rm $HOME/.ssh/id_rsa
+
+    # Run the external tests
+    make -C testing/external
+}
+
+failure() {
+    # Output each diag.log that contains failed test results.
+    for i in testing/btest/diag.log testing/external/bro-testing/diag.log; do
+        grep -qs '... failed$' $i && cat $i ;
+    done
+}
+
+# Coverity scan is run from a Travis CI cron job.
+if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
+    # Each Travis CI build consists of multiple jobs.  Here we choose one job
+    # to run the coverity scan.
+    JOB=`echo $TRAVIS_JOB_NUMBER | cut -d . -f 2`
+
+    if [ "$JOB" != "1" ]; then
+        echo "Coverity scan is performed only in the first job of this build"
+        exit 0
+    fi
+
+    # This is split up into two steps because the build outputs thousands of
+    # lines (which are collapsed into a single line on the web page).
+    if [ "$step" = "build" ]; then
+        build_coverity
+    elif [ "$step" = "run" ]; then
+        run_coverity
+    fi
+    exit 0
+fi
+
+# Run one step of a Travis CI job.  The "build" and "run" are split up into
+# separate steps because the build outputs thousands of lines (which are
+# collapsed into a single line on the web page).  The "failure" step is run
+# only when at least one test fails.
+if [ "$step" = "build" ]; then
+    build
+elif [ "$step" = "run" ]; then
+    run
+elif [ "$step" = "failure" ]; then
+    failure
+fi