mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
86 lines
1.8 KiB
Bash
Executable file
86 lines
1.8 KiB
Bash
Executable file
#! /usr/bin/env bash
|
|
#
|
|
# Downloads all traces as specified in <cwd>/traces.cfg to directory $1.
|
|
#
|
|
# traces.cfg must consist of lines of the form "<url> [<http-user>[:<http-password>]]"
|
|
|
|
if [ "$1" == "" ]; then
|
|
echo "usage: $(basename $0) <traces-directory>"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -e $cfg ]; then
|
|
echo "No $cfg found."
|
|
exit 1
|
|
fi
|
|
|
|
cfg=traces.cfg
|
|
|
|
for p in .proxy ../.proxy; do
|
|
if [ -e $p ]; then
|
|
proxy=$(cat $p | head -1 | awk '{print $1}')
|
|
echo Using proxy $proxy ...
|
|
proxy="ALL_PROXY=$proxy"
|
|
break
|
|
fi
|
|
done
|
|
|
|
mkdir -p $1
|
|
|
|
cat $cfg | while read line; do
|
|
|
|
if echo $line | grep -q '^[ \t]*$'; then
|
|
continue
|
|
fi
|
|
|
|
if echo $line | grep -q '^[ \t]*#'; then
|
|
continue
|
|
fi
|
|
|
|
url=$(echo $line | awk '{print $1}')
|
|
auth=$(echo $line | awk '{print $2}')
|
|
|
|
file=$1/$(echo $url | sed 's#^.*/##g')
|
|
fp=$file.md5sum
|
|
|
|
if [ "$auth" != "" ]; then
|
|
auth="-u $auth"
|
|
# Hide the hostname and directory names in output messages
|
|
safe_url=$(echo $url | sed 's#/[A-Za-z].*/#/[hidden]/#')
|
|
else
|
|
safe_url=$url
|
|
fi
|
|
|
|
# Get the fingerprint file.
|
|
echo Getting $safe_url.md5sum ...
|
|
eval "$proxy curl $auth -fsS --anyauth --retry 2 $url.md5sum -o $fp.tmp" || {
|
|
echo "Error: Could not get $safe_url.md5sum"
|
|
exit 1
|
|
}
|
|
echo
|
|
|
|
download=0
|
|
|
|
if [ -e $fp ]; then
|
|
if ! cmp -s $fp $fp.tmp; then
|
|
download=1
|
|
fi
|
|
else
|
|
download=1
|
|
fi
|
|
|
|
if [ "$download" = "1" ]; then
|
|
echo Getting $safe_url ...
|
|
eval "$proxy curl $auth --retry 2 -f --anyauth $url -o $file" || {
|
|
echo "Error: Could not get $safe_url"
|
|
exit 1
|
|
}
|
|
echo
|
|
mv $fp.tmp $fp
|
|
#else
|
|
# echo "`basename $file` already available."
|
|
fi
|
|
|
|
rm -f $fp.tmp
|
|
|
|
done
|