mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
3584495562
For trace files that require authentication to download, hide part of the URL in output messages. This avoids leaking potentially sensitive info when running tests using a continuous integration service.
84 lines
1.7 KiB
Bash
Executable file
84 lines
1.7 KiB
Bash
Executable file
#! /usr/bin/env bash
|
|
#
|
|
# Downloads all traces as specified in <cwd>/traces.cfg to directory $1.
|
|
#
|
|
# traces.cfg must consist of lines of the form "<url> [<http-user>[:<http-password>]]"
|
|
|
|
if [ "$1" == "" ]; then
|
|
echo "usage: `basename $0` <traces-directory>"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -e $cfg ]; then
|
|
echo "No $cfg found."
|
|
exit 1
|
|
fi
|
|
|
|
cfg=traces.cfg
|
|
|
|
for p in .proxy ../.proxy; do
|
|
if [ -e $p ]; then
|
|
proxy=`cat $p | head -1 | awk '{print $1}'`
|
|
echo Using proxy $proxy ...
|
|
proxy="ALL_PROXY=$proxy"
|
|
break
|
|
fi
|
|
done
|
|
|
|
mkdir -p $1
|
|
|
|
cat $cfg | while read line; do
|
|
|
|
if echo $line | grep -q '^[ \t]*$'; then
|
|
continue
|
|
fi
|
|
|
|
if echo $line | grep -q '^[ \t]*#'; then
|
|
continue
|
|
fi
|
|
|
|
url=`echo $line | awk '{print $1}'`
|
|
auth=`echo $line | awk '{print $2}'`
|
|
|
|
file=$1/`echo $url | sed 's#^.*/##g'`
|
|
fp=$file.md5sum
|
|
|
|
if [ "$auth" != "" ]; then
|
|
auth="-u $auth"
|
|
# Hide the hostname and directory names in output messages
|
|
safe_url=`echo $url | sed 's#/[A-Za-z].*/#/[hidden]/#'`
|
|
else
|
|
safe_url=$url
|
|
fi
|
|
|
|
# Get the fingerprint file.
|
|
if ! eval "$proxy curl $auth -fsS --anyauth $url.md5sum -o $fp.tmp"; then
|
|
echo "Error: Could not get $safe_url.md5sum, skipping download."
|
|
continue
|
|
fi
|
|
|
|
download=0
|
|
|
|
if [ -e $fp ]; then
|
|
if ! cmp -s $fp $fp.tmp; then
|
|
download=1
|
|
fi
|
|
else
|
|
download=1
|
|
fi
|
|
|
|
if [ "$download" = "1" ]; then
|
|
echo Getting $safe_url ...
|
|
echo
|
|
eval "$proxy curl $auth -f --anyauth $url -o $file"
|
|
echo
|
|
mv $fp.tmp $fp
|
|
#else
|
|
# echo "`basename $file` already available."
|
|
fi
|
|
|
|
rm -f $fp.tmp
|
|
|
|
done
|
|
|
|
|