diff --git a/auxil/zeek-aux/CMakeLists.txt b/auxil/zeek-aux/CMakeLists.txt new file mode 100644 index 0000000000..4778956377 --- /dev/null +++ b/auxil/zeek-aux/CMakeLists.txt @@ -0,0 +1,49 @@ +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) + +# ############################################################################## +# Dependency Configuration + +find_package(PCAP REQUIRED) + +include_directories(BEFORE ${PCAP_INCLUDE_DIR}) + +# ############################################################################## +# System Introspection + +include(CheckHeaders) +include(CheckFunctions) +include(CheckNameserCompat) +include(MiscTests) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) + +include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}) + +# ############################################################################## +# Recurse on sub-directories + +# For binary packaging or if this is the main CMake project, go through the +# regular install target, else use a custom target so programs have to be +# explicitly installed by the user via "make install-aux" +macro (AddAuxInstallTarget _target) + add_custom_target( + install-${_target} COMMAND ${CMAKE_COMMAND} -E make_directory + ${CMAKE_INSTALL_PREFIX}/bin + COMMAND ${CMAKE_COMMAND} -E copy $ ${CMAKE_INSTALL_PREFIX}/bin) + add_dependencies(install-${_target} ${_target}) + set(AUX_TARGETS install-${_target};${AUX_TARGETS}) + set(AUX_TARGETS ${AUX_TARGETS} PARENT_SCOPE) +endmacro (AddAuxInstallTarget) + +if (NOT ZEEK_MAN_INSTALL_PATH) + set(ZEEK_MAN_INSTALL_PATH ${CMAKE_INSTALL_PREFIX}/share/man) +endif () + +add_subdirectory(adtrace) +add_subdirectory(zeek-archiver) +add_subdirectory(zeek-cut) +add_subdirectory(rst) + +add_custom_target(install-aux + COMMENT "Zeek auxiliary tools installed to ${CMAKE_INSTALL_PREFIX}/bin") +add_dependencies(install-aux ${AUX_TARGETS}) diff --git a/auxil/zeek-aux/README b/auxil/zeek-aux/README new file mode 100644 index 0000000000..15af4f6e99 --- /dev/null +++ b/auxil/zeek-aux/README @@ -0,0 +1,107 @@ +.. -*- mode: rst; -*- +.. +.. Version number is filled in automatically. +.. |version| replace:: 0.50-174 + +======================= +Zeek Auxiliary Programs +======================= + +.. contents:: + +:Version: |version| + +Handy auxiliary programs related to the use of the Zeek Network Security +Monitor (https://www.zeek.org). + +Installation +============ + +Installation is simple and standard:: + + ./configure + make + make install + +adtrace +======= + +The "adtrace" utility is used to compute the +network address that compose the internal and extern nets that Zeek +is monitoring. This program just reads a pcap +(tcpdump) file and writes out the src MAC, dst MAC, src IP, dst +IP for each packet seen in the file. + +zeek-archiver +============= + +This is a modern replacement for Zeek's historical log-archival process. For +details, please refer to its dedicated README in the zeek-archiver subdirectory. + +zeek-cut +======== + +The "zeek-cut" utility reads ASCII Zeek logs on standard input +and outputs them to standard output with only the specified columns (the +column names can be found in each log file in the "#fields" header line). +If no column names are specified, then "zeek-cut" simply outputs all columns. + +There are several command-line options available to modify the output (run +"zeek-cut -h" to see a list of all options). There are options to convert +timestamps into human-readable format, and options to specify whether or not +to include the format header lines in the output (by default, they're not +included). + +For example, the following command will output the three specified columns +from conn.log with the timestamps from the "ts" column being converted to +human-readable format:: + + cat conn.log | zeek-cut -d ts id.orig_h id.orig_p + +The specified order of the column names determines the output order of the +columns (i.e., "zeek-cut" can reorder the columns). + +The "zeek-cut" utility can read the concatenation of one or more uncompressed +ASCII log files (however, JSON format is not supported) produced by Zeek +version 2.0 or newer, as long as each log file contains format header +lines (these are the lines at the beginning of the file starting with "#"). +In fact, "zeek-cut" can process the concatenation of multiple ASCII log files +that have different column layouts. + +To read a compressed log file, a tool such as "zcat" must be used to +uncompress the file. For example, "zeek-cut" can read a group of compressed +conn.log files with a command like this:: + + zcat conn.*.log.gz | zeek-cut + + +devel-tools +=========== + +A set of scripts used commonly for Zeek development. Note that none of +these scripts are installed by 'make install'. + +extract-conn-by-uid + Extracts a connection from a trace file based + on its UID found in Zeek's conn.log + +gen-mozilla-ca-list.rb + Generates list of Mozilla SSL root certificates in + a format readable by Zeek. + +update-changes + A script to maintain the CHANGES and VERSION files. + +git-show-fastpath + Show commits to the fastpath branch not yet merged into master. + +cpu-bench-with-trace + Run a number of Zeek benchmarks on a trace file. + + +rst +=== + +The "rst" utility can be invoked by a Zeek script to terminate an +established TCP connection by forging RST tear-down packets. + diff --git a/auxil/zeek-aux/README.rst b/auxil/zeek-aux/README.rst new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/auxil/zeek-aux/README.rst @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/auxil/zeek-aux/adtrace/CMakeLists.txt b/auxil/zeek-aux/adtrace/CMakeLists.txt new file mode 100644 index 0000000000..f26f257c27 --- /dev/null +++ b/auxil/zeek-aux/adtrace/CMakeLists.txt @@ -0,0 +1,8 @@ +find_package(PCAP REQUIRED) + +set(adtrace_SRCS adtrace.c) + +add_executable(adtrace ${adtrace_SRCS}) +target_include_directories(adtrace BEFORE PRIVATE ${PCAP_INCLUDE_DIR}) +target_link_libraries(adtrace ${PCAP_LIBRARY}) +AddAuxInstallTarget(adtrace) diff --git a/auxil/zeek-aux/adtrace/adtrace.c b/auxil/zeek-aux/adtrace/adtrace.c new file mode 100644 index 0000000000..0faae96039 --- /dev/null +++ b/auxil/zeek-aux/adtrace/adtrace.c @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ip.h" +#include "ether.h" +#include "ethertype.h" + +pcap_t *p; + +const u_char* printEAddr(const u_char* pkt, u_char* endp){ + const struct ether_header *ep; + int i=0; + ep = (const struct ether_header*) pkt; + + if (pkt+ETHER_HDRLEN > endp || + ntohs(ep->ether_type) != ETHERTYPE_IP){ + return 0; + } + + for (i = 0; i0) putchar(':'); + printf("%02x", ep->ether_shost[i]); + } + putchar (' '); + for (i = 0; i0) putchar(':'); + printf("%02x", ep->ether_dhost[i]); + } + putchar(' '); + return (pkt+ETHER_HDRLEN); +} + +void printIPAddr(const u_char* pkt, u_char* endp){ + const struct ip* iph; + if (pkt+sizeof(struct ip) > endp) return; + iph = (const struct ip*) pkt; + fputs ((char*) inet_ntoa(iph->ip_src), stdout); + putchar(' '); + puts ((char*) inet_ntoa(iph->ip_dst)); +} + +void handler(u_char *user, const struct pcap_pkthdr *head, const u_char *packet){ + u_char* endp; + + endp =(u_char*) packet + head->caplen; + packet = printEAddr(packet, endp); + if (packet) + printIPAddr(packet, endp); +} + +void usage(char *av[]) +{ + fprintf(stderr,"usage: %s filename \n", av[0]); + exit(1); +} + +int main (int argc, char *argv[]) +{ + char *file; + char errbuf[PCAP_ERRBUF_SIZE]; + u_char* pkt, endp; + struct pcap_pkthdr *head; + + if ( argc != 2 ) + usage(argv); + + file = argv[1]; + + p = pcap_open_offline(file, errbuf); + if(p==NULL){ + fprintf (stderr, "cannot open %s: %s\n", file, errbuf); + exit(2); + } + + if (pcap_datalink(p) != DLT_EN10MB){ + fputs ("sorry, currently only ethernet links supported\n", stderr); + exit(1); //if it is not ethernet we are watching we won't have MACs + } + + pcap_loop(p, -1, handler, NULL); + pcap_close(p); + return(0); +} + diff --git a/auxil/zeek-aux/adtrace/ether.h b/auxil/zeek-aux/adtrace/ether.h new file mode 100644 index 0000000000..77d0377945 --- /dev/null +++ b/auxil/zeek-aux/adtrace/ether.h @@ -0,0 +1,59 @@ +/* @(#) $Header$ (LBL) */ +/* + * Copyright (c) 1982, 1986, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)if_ether.h 8.3 (Berkeley) 5/2/95 + */ + +#define ETHERMTU 1500 + +/* + * The number of bytes in an ethernet (MAC) address. + */ +#define ETHER_ADDR_LEN 6 + +/* + * Structure of a DEC/Intel/Xerox or 802.3 Ethernet header. + */ +struct ether_header { + u_int8_t ether_dhost[ETHER_ADDR_LEN]; + u_int8_t ether_shost[ETHER_ADDR_LEN]; + u_int16_t ether_type; +}; + +/* + * Length of a DEC/Intel/Xerox or 802.3 Ethernet header; note that some + * compilers may pad "struct ether_header" to a multiple of 4 bytes, + * for example, so "sizeof (struct ether_header)" may not give the right + * answer. + */ +#define ETHER_HDRLEN 14 diff --git a/auxil/zeek-aux/adtrace/ethertype.h b/auxil/zeek-aux/adtrace/ethertype.h new file mode 100644 index 0000000000..1f6aab6776 --- /dev/null +++ b/auxil/zeek-aux/adtrace/ethertype.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 1993, 1994, 1996 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: (1) source code distributions + * retain the above copyright notice and this paragraph in its entirety, (2) + * distributions including binary code include the above copyright notice and + * this paragraph in its entirety in the documentation or other materials + * provided with the distribution, and (3) all advertising materials mentioning + * features or use of this software display the following acknowledgement: + * ``This product includes software developed by the University of California, + * Lawrence Berkeley Laboratory and its contributors.'' Neither the name of + * the University nor the names of its contributors may be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + * + * @(#) $Header$ (LBL) + */ + +/* + * Ethernet types. + * + * We wrap the declarations with #ifdef, so that if a file includes + * , which may declare some of these, we don't + * get a bunch of complaints from the C compiler about redefinitions + * of these values. + * + * We declare all of them here so that no file has to include + * if all it needs are ETHERTYPE_ values. + */ + +#ifndef ETHERTYPE_PUP +#define ETHERTYPE_PUP 0x0200 /* PUP protocol */ +#endif +#ifndef ETHERTYPE_IP +#define ETHERTYPE_IP 0x0800 /* IP protocol */ +#endif +#ifndef ETHERTYPE_ARP +#define ETHERTYPE_ARP 0x0806 /* Addr. resolution protocol */ +#endif +#ifndef ETHERTYPE_REVARP +#define ETHERTYPE_REVARP 0x8035 /* reverse Addr. resolution protocol */ +#endif +#ifndef ETHERTYPE_NS +#define ETHERTYPE_NS 0x0600 +#endif +#ifndef ETHERTYPE_SPRITE +#define ETHERTYPE_SPRITE 0x0500 +#endif +#ifndef ETHERTYPE_TRAIL +#define ETHERTYPE_TRAIL 0x1000 +#endif +#ifndef ETHERTYPE_MOPDL +#define ETHERTYPE_MOPDL 0x6001 +#endif +#ifndef ETHERTYPE_MOPRC +#define ETHERTYPE_MOPRC 0x6002 +#endif +#ifndef ETHERTYPE_DN +#define ETHERTYPE_DN 0x6003 +#endif +#ifndef ETHERTYPE_LAT +#define ETHERTYPE_LAT 0x6004 +#endif +#ifndef ETHERTYPE_SCA +#define ETHERTYPE_SCA 0x6007 +#endif +#ifndef ETHERTYPE_REVARP +#define ETHERTYPE_REVARP 0x8035 +#endif +#ifndef ETHERTYPE_LANBRIDGE +#define ETHERTYPE_LANBRIDGE 0x8038 +#endif +#ifndef ETHERTYPE_DECDNS +#define ETHERTYPE_DECDNS 0x803c +#endif +#ifndef ETHERTYPE_DECDTS +#define ETHERTYPE_DECDTS 0x803e +#endif +#ifndef ETHERTYPE_VEXP +#define ETHERTYPE_VEXP 0x805b +#endif +#ifndef ETHERTYPE_VPROD +#define ETHERTYPE_VPROD 0x805c +#endif +#ifndef ETHERTYPE_ATALK +#define ETHERTYPE_ATALK 0x809b +#endif +#ifndef ETHERTYPE_AARP +#define ETHERTYPE_AARP 0x80f3 +#endif +#ifndef ETHERTYPE_8021Q +#define ETHERTYPE_8021Q 0x8100 +#endif +#ifndef ETHERTYPE_IPX +#define ETHERTYPE_IPX 0x8137 +#endif +#ifndef ETHERTYPE_IPV6 +#define ETHERTYPE_IPV6 0x86dd +#endif +#ifndef ETHERTYPE_PPP +#define ETHERTYPE_PPP 0x880b +#endif +#ifndef ETHERTYPE_MPLS +#define ETHERTYPE_MPLS 0x8847 +#endif +#ifndef ETHERTYPE_MPLS_MULTI +#define ETHERTYPE_MPLS_MULTI 0x8848 +#endif +#ifndef ETHERTYPE_PPPOED +#define ETHERTYPE_PPPOED 0x8863 +#endif +#ifndef ETHERTYPE_PPPOES +#define ETHERTYPE_PPPOES 0x8864 +#endif +#ifndef ETHERTYPE_LOOPBACK +#define ETHERTYPE_LOOPBACK 0x9000 +#endif diff --git a/auxil/zeek-aux/adtrace/ip.h b/auxil/zeek-aux/adtrace/ip.h new file mode 100644 index 0000000000..3d930537c9 --- /dev/null +++ b/auxil/zeek-aux/adtrace/ip.h @@ -0,0 +1,159 @@ +/* @(#) $Header$ (LBL) */ +/* + * Copyright (c) 1982, 1986, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)ip.h 8.2 (Berkeley) 6/1/94 + */ + +/* + * Definitions for internet protocol version 4. + * Per RFC 791, September 1981. + */ +#define IPVERSION 4 + +/* + * Structure of an internet header, naked of options. + * + * We declare ip_len and ip_off to be short, rather than u_short + * pragmatically since otherwise unsigned comparisons can result + * against negative integers quite easily, and fail in subtle ways. + */ +struct ip { + u_int8_t ip_vhl; /* header length, version */ +#define IP_V(ip) (((ip)->ip_vhl & 0xf0) >> 4) +#define IP_HL(ip) ((ip)->ip_vhl & 0x0f) + u_int8_t ip_tos; /* type of service */ + u_int16_t ip_len; /* total length */ + u_int16_t ip_id; /* identification */ + u_int16_t ip_off; /* fragment offset field */ +#define IP_DF 0x4000 /* dont fragment flag */ +#define IP_MF 0x2000 /* more fragments flag */ +#define IP_OFFMASK 0x1fff /* mask for fragmenting bits */ + u_int8_t ip_ttl; /* time to live */ + u_int8_t ip_p; /* protocol */ + u_int16_t ip_sum; /* checksum */ + struct in_addr ip_src,ip_dst; /* source and dest address */ +}; + +#define IP_MAXPACKET 65535 /* maximum packet size */ + +/* + * Definitions for IP type of service (ip_tos) + */ +#define IPTOS_LOWDELAY 0x10 +#define IPTOS_THROUGHPUT 0x08 +#define IPTOS_RELIABILITY 0x04 + +/* + * Definitions for IP precedence (also in ip_tos) (hopefully unused) + */ +#define IPTOS_PREC_NETCONTROL 0xe0 +#define IPTOS_PREC_INTERNETCONTROL 0xc0 +#define IPTOS_PREC_CRITIC_ECP 0xa0 +#define IPTOS_PREC_FLASHOVERRIDE 0x80 +#define IPTOS_PREC_FLASH 0x60 +#define IPTOS_PREC_IMMEDIATE 0x40 +#define IPTOS_PREC_PRIORITY 0x20 +#define IPTOS_PREC_ROUTINE 0x00 + +/* + * Definitions for options. + */ +#define IPOPT_COPIED(o) ((o)&0x80) +#define IPOPT_CLASS(o) ((o)&0x60) +#define IPOPT_NUMBER(o) ((o)&0x1f) + +#define IPOPT_CONTROL 0x00 +#define IPOPT_RESERVED1 0x20 +#define IPOPT_DEBMEAS 0x40 +#define IPOPT_RESERVED2 0x60 + +#define IPOPT_EOL 0 /* end of option list */ +#define IPOPT_NOP 1 /* no operation */ + +#define IPOPT_RR 7 /* record packet route */ +#define IPOPT_TS 68 /* timestamp */ +#define IPOPT_SECURITY 130 /* provide s,c,h,tcc */ +#define IPOPT_LSRR 131 /* loose source route */ +#define IPOPT_SATID 136 /* satnet id */ +#define IPOPT_SSRR 137 /* strict source route */ + +/* + * Offsets to fields in options other than EOL and NOP. + */ +#define IPOPT_OPTVAL 0 /* option ID */ +#define IPOPT_OLEN 1 /* option length */ +#define IPOPT_OFFSET 2 /* offset within option */ +#define IPOPT_MINOFF 4 /* min value of above */ + +/* + * Time stamp option structure. + */ +struct ip_timestamp { + u_int8_t ipt_code; /* IPOPT_TS */ + u_int8_t ipt_len; /* size of structure (variable) */ + u_int8_t ipt_ptr; /* index of current entry */ + u_int8_t ipt_oflwflg; /* flags, overflow counter */ +#define IPTS_OFLW(ip) (((ipt)->ipt_oflwflg & 0xf0) >> 4) +#define IPTS_FLG(ip) ((ipt)->ipt_oflwflg & 0x0f) + union ipt_timestamp { + u_int32_t ipt_time[1]; + struct ipt_ta { + struct in_addr ipt_addr; + u_int32_t ipt_time; + } ipt_ta[1]; + } ipt_timestamp; +}; + +/* flag bits for ipt_flg */ +#define IPOPT_TS_TSONLY 0 /* timestamps only */ +#define IPOPT_TS_TSANDADDR 1 /* timestamps and addresses */ +#define IPOPT_TS_PRESPEC 3 /* specified modules only */ + +/* bits for security (not byte swapped) */ +#define IPOPT_SECUR_UNCLASS 0x0000 +#define IPOPT_SECUR_CONFID 0xf135 +#define IPOPT_SECUR_EFTO 0x789a +#define IPOPT_SECUR_MMMM 0xbc4d +#define IPOPT_SECUR_RESTR 0xaf13 +#define IPOPT_SECUR_SECRET 0xd788 +#define IPOPT_SECUR_TOPSECRET 0x6bc5 + +/* + * Internet implementation parameters. + */ +#define MAXTTL 255 /* maximum time to live (seconds) */ +#define IPDEFTTL 64 /* default ttl, from RFC 1340 */ +#define IPFRAGTTL 60 /* time to live for frags, slowhz */ +#define IPTTLDEC 1 /* subtracted when forwarding */ + +#define IP_MSS 576 /* default maximum segment size */ diff --git a/auxil/zeek-aux/config.h.in b/auxil/zeek-aux/config.h.in new file mode 100644 index 0000000000..9d9d31bc1b --- /dev/null +++ b/auxil/zeek-aux/config.h.in @@ -0,0 +1,28 @@ +/* These autoconf variables are obsolete; it's portable to assume C89 + and signal handlers that return void */ +#define RETSIGTYPE void +#define RETSIGVAL + +/* Define if you have the `sigaction' function. */ +#cmakedefine HAVE_SIGACTION + +/* Define if you have the `sigset' function. */ +#cmakedefine HAVE_SIGSET + +/* Define if you have the header file. */ +#cmakedefine HAVE_MEMORY_H + +/* Define if you have the `strerror' function. */ +#cmakedefine HAVE_STRERROR + +/* should explicitly declare socket() and friends */ +#cmakedefine DO_SOCK_DECL + +/* Compatibility for Darwin */ +#cmakedefine NEED_NAMESER_COMPAT_H + +/* have os-proto.h */ +#cmakedefine HAVE_OS_PROTO_H + +/* have sin_len field in sockaddr_in */ +#cmakedefine SIN_LEN diff --git a/auxil/zeek-aux/devel-tools/check-release b/auxil/zeek-aux/devel-tools/check-release new file mode 100755 index 0000000000..85b2704906 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/check-release @@ -0,0 +1,15 @@ +#! /usr/bin/env bash +# +# Checks the current repository and all if its submodules for being +# in "release shape". + +repo_status="$(cd $(dirname $0) && pwd)/repo-status" +check_cmd="test -e CHANGES && $repo_status" + +printf ' %20s %-10s %-8s %-8s %-7s %-5s %-15s %s\n' "" "Branch" "CHANGES" "Pending" "Modif" "Sub" "VERSION" "Tags" + +( + eval $check_cmd + git submodule foreach -q --recursive "$check_cmd; true" +) | + awk '/!/{print "- ", $0; next;} {print "+ ", $0; next;}' diff --git a/auxil/zeek-aux/devel-tools/cpu-bench-with-trace b/auxil/zeek-aux/devel-tools/cpu-bench-with-trace new file mode 100755 index 0000000000..c2bc1d9f57 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/cpu-bench-with-trace @@ -0,0 +1,32 @@ +#! /usr/bin/env bash +# +# This runs a number of Zeek configurations on trace $2. It +# starts with the bare config and then +# kept adding the scripts load from init-default.zeek and local.zeek one +# by one, measuring user time for each run (i.e., the measurements are +# cumulative). + +if [ "$2" == "" ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +zeek=$1 +trace=$2 +tmp=/tmp/bench.$$.zeek + +export ZEEKPATH=$($zeek/build/zeek-path-dev) + +cat $tmp + +cat $zeek/scripts/base/init-default.zeek $zeek/scripts/site/local.zeek | grep '^ *@load' | while read line; do + echo $line >>$tmp + script=$(echo $line | awk '{print $2}' | sed 's#/#.#g') + output="bench.output.$script.log" + + (time -p $zeek/build/src/zeek -b -r $trace $tmp) >$output 2>&1 + user=$(cat $output | grep user | awk '{print $2}') + printf "%40s %s\n" $script $user +done + +rm -f $tmp diff --git a/auxil/zeek-aux/devel-tools/extract-conn-by-uid b/auxil/zeek-aux/devel-tools/extract-conn-by-uid new file mode 100755 index 0000000000..7fec499f47 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/extract-conn-by-uid @@ -0,0 +1,42 @@ +#! /usr/bin/env bash +# +# Searches the connection with UID $1 in conn.log, and then extracts +# it from trace file $2. + +if [ $# != 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +uid=$1 +trace=$2 + +if [ ! -e conn.log ]; then + echo "no conn.log found" + exit 1 +fi + +if [ ! -e $trace ]; then + echo "trace $trace not found" + exit 1 +fi + +filter=$(awk -v uid=$uid '$2==uid { printf("host %s and port %s and host %s and port %s\n", $3, $4, $5, $6)}' new(); +my $google_known_logs_json = $ua->get($google_log_url); +croak("Could not get $google_log_url") unless defined($google_known_logs_json); + +my $list = parse_json($google_known_logs_json->content); + +say "#\n# Do not edit this file. This file is automatically generated by gen-ct-list.pl"; +say "# File generated at ".localtime; +say "# File generated from ".$google_log_url; +say "# Source file generated at: ".$list->{log_list_timestamp}; +say "# Source file version: ".$list->{version}; +say "#"; +say ""; +say '@load base/protocols/ssl'; +say "module SSL;"; +say ""; +say '## @docs-omit-value'; +say "redef ct_logs += {"; + +for my $operator (@{$list->{operators}}) { + my $opname = $operator->{name}; + for my $log (@{$operator->{logs}}) { + my $key = join('', map {"\\x$_" } unpack("(H2)*", decode_base64($log->{key}))); + my $logid = join('', map {"\\x$_" } unpack("(H2)*", sha256(decode_base64($log->{key})))); + my $mmd = $log->{mmd}; + my $url = $log->{url}; + my $desc = $log->{description}; + say "[\"$logid\"] = CTInfo(\$description=\"$desc\", \$operator=\"$opname\", \$url=\"$url\", \$maximum_merge_delay=$mmd, \$key=\"$key\"),"; + } +} + +say "};"; diff --git a/auxil/zeek-aux/devel-tools/gen-mozilla-ca-list.rb b/auxil/zeek-aux/devel-tools/gen-mozilla-ca-list.rb new file mode 100755 index 0000000000..ffe41e0f80 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/gen-mozilla-ca-list.rb @@ -0,0 +1,84 @@ +#!/usr/bin/env ruby + +tmpcert = "/tmp/tmpcert.der" + +incert=false +intrust=false + +if ARGV.length != 1 + abort "\nPass path to the certdata.txt you want to add as first input argument to this script\n\n"+ + "certdata.txt can be retrieved from the newest NSS release." +end + +url = 'http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/ckfw/builtins/certdata.txt?raw=1' +io = open(ARGV[0]); + +puts "# Don't edit! This file is automatically generated." +puts "# Generated at: #{Time.now}" +puts "# Generated from: #{url}" +puts "#" +puts "# The original source file comes with this licensing statement:" +puts "#" +puts "# This Source Code Form is subject to the terms of the Mozilla Public" +puts "# License, v. 2.0. If a copy of the MPL was not distributed with this" +puts "# file, You can obtain one at http://mozilla.org/MPL/2.0/." +puts "" + +puts "@load base/protocols/ssl" +puts "module SSL;"; +puts ""; +puts "## @docs-omit-value" +puts "redef root_certs += {"; + +all_certs = [] +all_subjects = [] + +cert_name = "" +cert = "" +io.each do |line| + line.chomp! + if intrust + if line =~ /^CKA_TRUST_SERVER_AUTH/ + if line =~ /CKT_NSS_TRUSTED_DELEGATOR/ + File.open(tmpcert, "wb") do |f| + byteArray = cert.split("\\x") + max = byteArray.length() - 1 + byteArray[1..max].each do | byte | + f.print byte.hex.chr + end + end + + cert_subj = `openssl x509 -in #{tmpcert} -inform DER -noout -subject -nameopt RFC2253` + cert_subj["subject="]= "" + cert_subj.chomp! + File.delete(tmpcert) + + if not all_subjects.include?(cert_subj) + puts " [\"#{cert_subj}\"] = \"#{cert}\"," + all_subjects << cert_subj + end + end + intrust=false + end + else + if line =~ /^CKA_LABEL/ + cert_name = line.sub(/.*\"(.*)\".*/, "\\1") + i = 0 + while all_certs.include?(cert_name) + i+=1 + cert_name += " #{i}" + end + all_certs << cert_name + elsif line =~ /^CKA_VALUE MULTILINE_OCTAL/ + incert=true + cert="" + elsif line =~ /^CKA_CLASS CK_OBJECT_CLASS CKO_NSS_TRUST/ + intrust=true + elsif line =~ /^END/ + incert=false + elsif incert + cert += line.split(/\\/).collect { |x| x.oct.chr.unpack("H2")[0].upcase if x!="" }.join("\\x") + end + end +end +puts "};" diff --git a/auxil/zeek-aux/devel-tools/git-delete-old-branches b/auxil/zeek-aux/devel-tools/git-delete-old-branches new file mode 100755 index 0000000000..60263a098c --- /dev/null +++ b/auxil/zeek-aux/devel-tools/git-delete-old-branches @@ -0,0 +1,32 @@ +# ! /usr/bin/env bash +# +# Adapted from http://devblog.springest.com/a-script-to-remove-old-git-branches +# + +# This has to be run from master +git checkout master + +# Update our list of remotes +git fetch +git remote prune origin + +# # Remove local fully merged branches +# git branch --merged master | grep -v 'master$' | xargs git branch -d + +# Show remote fully merged branches +echo "The following remote branches are fully merged and will be removed:" +git branch -r --merged master | sed 's/ *origin\///' | grep '^topic/' + +if [ $? != 0 ]; then + exit 0 +fi + +read -p "Continue (y/n)? " + +if [ "$REPLY" == "y" ] +then + # Remove remote fully merged branches + git branch -r --merged master | sed 's/ *origin\///' \ + | grep '^topic/' | xargs -I% git push origin :% + echo "Done!" +fi diff --git a/auxil/zeek-aux/devel-tools/git-move-submodules b/auxil/zeek-aux/devel-tools/git-move-submodules new file mode 100755 index 0000000000..fa985b4b06 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/git-move-submodules @@ -0,0 +1,77 @@ +#! /usr/bin/env bash +# +# Recursively check outs the most recent version of all submodules on a given +# branch, and commits the updates to the parents. + +branch=$1 + +if [ "$branch" == "" ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +paths_to_push=() + +function update_module { + local cwd=$1 + local i + local modules="" + + cd $cwd + + # These submodules should be ignored by the loop below. + local ignored_modules=( + "3rdparty" + "IXWebSocket" + "c-ares" + "caf" + "cppzmq" + "expected-lite" + "filesystem" + "highwayhash" + "libkqueue" + "libunistd" + "out_ptr" + "prometheus-cpp" + "rapidjson" + "vcpkg") + + # Note we don't use --recursive here, as we want to do a depth-first + # search so that we update children first. + for i in $(git submodule foreach -q 'echo $path' | grep -vE $( + IFS="|" + echo "${ignored_modules[*]}" + )); do + # See if repository has a branch of the given name. Otherwise leave it alone. + (cd $i && git show-ref --verify --quiet refs/heads/$branch) || continue + + modules="$modules $i" + + echo "--- Checking out $branch of $(basename $i)" + cd $i + git fetch -q || exit 1 + git checkout -q $branch || exit 1 + git merge origin/master || exit 1 + + update_module $cwd/$i + + cd $cwd + done + + if [ "$modules" != "" ]; then + if [ -n "$(git status --untracked-files=no --porcelain)" ]; then + echo "+++ Committing updates to $(basename $cwd)" + git commit -m 'Updating submodule(s) [nomail]' --only $modules + paths_to_push+=($cwd) + fi + fi + +} + +update_module $(pwd) + +echo +echo "Added ${#paths_to_push[@]} commits. Run the following commands to push them:" +for path in "${paths_to_push[@]}"; do + echo "(cd ${path} && git push)" +done diff --git a/auxil/zeek-aux/devel-tools/git-pre-push-hook b/auxil/zeek-aux/devel-tools/git-pre-push-hook new file mode 100755 index 0000000000..286d60b601 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/git-pre-push-hook @@ -0,0 +1,31 @@ +#!/bin/sh +# +# A pre-push hook that makes sure all testing/external changes +# have been pushed already. If not, it will abort. Note that +# it will only check for unpushed commits, not for uncommitted +# changes. +# +# To install this, copy it into you Zeek tree's .git/hooks/pre-push. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If this script exits with a non-zero status nothing will be pushed. + +test -d testing/external || exit 0 + +cd testing/external + +base=$(pwd) +abort=0 + +for repo in $(./scripts/find-git-repos); do + cd ${base}/${repo} && + git rev-list @{u}.. | grep -q . && + echo "ERROR: testing/external/$(basename $repo) has commits that are not pushed." && + abort=1 +done + +exit ${abort} diff --git a/auxil/zeek-aux/devel-tools/git-show-fastpath b/auxil/zeek-aux/devel-tools/git-show-fastpath new file mode 100755 index 0000000000..e279aeb24e --- /dev/null +++ b/auxil/zeek-aux/devel-tools/git-show-fastpath @@ -0,0 +1,19 @@ +#! /usr/bin/env bash +# +# Shows pending fastpath commits for all modules. + +show="git show-ref -q origin/fastpath && git --no-pager log --format=oneline origin/fastpath ^master || exit 0" + +(echo "Entering " && eval $show && git submodule foreach --recursive "$show") | awk ' + +/Entering/ { current = $2; next } + +{ + if ( current != "" ) + print "==" current; + + print; + + current = ""; +} +' diff --git a/auxil/zeek-aux/devel-tools/github-manage b/auxil/zeek-aux/devel-tools/github-manage new file mode 100755 index 0000000000..8a81a32d48 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/github-manage @@ -0,0 +1,461 @@ +#! /usr/bin/env python3 +# +# https://pypi.org/project/argcomplete/#global-completion +# PYTHON_ARGCOMPLETE_OK + +"""This script provides tooling to administrate the github.com/zeek organization.""" + +import argparse +import json +import os +import sys +from enum import IntEnum +from pathlib import Path + +# This is the high-level error for API problems, per +# https://github.com/fastai/ghapi/issues/138 and confirmed +# in manual testing +from urllib.error import HTTPError + + +def print_err(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +try: + import yaml +except ImportError: + # PyYAML is optional; we will see below whether we need it as we retrieve + # the auth token. + pass + +try: + from ghapi.all import GhApi, paged +except ImportError: + print_err("This requires the Python Github API package: https://ghapi.fast.ai") + sys.exit(1) + +try: + # Argcomplete provides command-line completion for users of argparse. + # We support it if available, but don't complain when it isn't. + import argcomplete +except ImportError: + pass + +GITHUB_SERVER = "github.com" +GITHUB_ORG = "zeek" + +# There's a mismatch between how Github shows access levels in the UI vs how +# they work in the API. We use the UI-level ones here and provide helper +# functions to translate. The discrepancies are "read" being "pull" in the API, +# and "write" being "push". The others match. +AccessLevel = IntEnum("AccessLevel", ["READ", "TRIAGE", "WRITE", "MAINTAIN", "ADMIN"]) + + +def get_api_token(args): + if args.auth_token: + return args.auth_token + + # This is the ghapi-recommended way to configure your token and works + # transparently: + token = os.getenv("GITHUB_TOKEN") + if token: + return token + + # Try somewhat hard to locate gh's config file: + gh_host_config = ( + Path(os.getenv("XDG_CONFIG_HOME", default=Path.home() / ".config")) + / "gh" + / "hosts.yml" + ) + if gh_host_config.is_file(): + if "yaml" not in sys.modules: + print_err( + "This requires PyYAML to use your existing gh config. See: https://pyyaml.org" + ) + print_err( + "Please set the GITHUB_TOKEN environment variable to your Github API token." + ) + sys.exit(1) + + try: + with open(gh_host_config) as stream: + ymldata = yaml.safe_load(stream) + return ymldata[GITHUB_SERVER]["oauth_token"] + except (OSError, yaml.YAMLError, KeyError) as error: + print_err(f"Unexpected gh configuration ({error}).") + print_err( + "Please set the GITHUB_TOKEN environment variable to your Github API token." + ) + sys.exit(1) + + print_err( + "Please provide a Github API token via --auth-token or set the GITHUB_TOKEN environment variable" + ) + sys.exit(1) + + +def get_access_level_string(perms): + res = get_access_level(perms) + if not res: + return None + return res.name.lower() + + +def access_level_to_github(level): + if level == AccessLevel.ADMIN: + return "admin" + if level == AccessLevel.MAINTAIN: + return "maintain" + if level == AccessLevel.WRITE: + return "push" + if level == AccessLevel.TRIAGE: + return "triage" + if level == AccessLevel.READ: + return "pull" + return None + + +def access_level_from_github(level): + if level == "admin": + return AccessLevel.ADMIN + if level == "maintain": + return AccessLevel.MAINTAIN + if level == "push": + return AccessLevel.WRITE + if level == "triage": + return AccessLevel.TRIAGE + if level == "pull": + return AccessLevel.READ + return None + + +def get_access_level(perm): + """Returns the access level for a given user/team as shown on the access + settings page of a repo, based on the given set of permissions. This returns + an AccessLevel, or None when no access level was identifiable. + """ + # This is a bit messy because permissions are presented differently in + # different corners of the API. Sometimes it's a string, sometimes an + # object with individual permission bits. For now just check if it's a + # string and handle that separately. + if isinstance(perm, str): + return access_level_from_github(perm) + + try: + if perm.admin: + return AccessLevel.ADMIN + if perm.maintain: + return AccessLevel.MAINTAIN + if perm.push: + return AccessLevel.WRITE + if perm.triage: + return AccessLevel.TRIAGE + if perm.pull: + return AccessLevel.READ + except AttributeError: + pass + + return None + + +def get_all_repos(api): + """Returns a list of all repos in the GITHUB_ORG, sorted by name.""" + repos = [] + + for page in paged(api.repos.list_for_org, org=GITHUB_ORG, per_page=100): + repos.extend(page) + + repos.sort(key=lambda repo: repo.name) + return repos + + +def cmd_repos_list_access(api, args): + result = [] + + for repo in get_all_repos(api): + teams = api.repos.list_teams(repo.name) + teams_results = [] + for team in sorted(teams, key=lambda t: t.name): + teams_results.append( + { + "name": team.name, + "access": get_access_level_string(team.permissions), + } + ) + + # "direct" here means we don't list each individual member of teams that + # also have access to the repo: + users = api.repos.list_collaborators(repo.name, affiliation="direct") + users_results = [] + for user in sorted(users, key=lambda u: u.login): + users_results.append( + { + "name": user.login, + "access": get_access_level_string(user.permissions), + } + ) + + result.append( + { + "repo": repo.name, + "teams": teams_results, + "users": users_results, + } + ) + + print(json.dumps(result, sort_keys=True)) + return True + + +def cmd_repos_list_redundant_users(api, args): + result = [] + + for repo in get_all_repos(api): + teams = api.repos.list_teams(repo.name) + teams_data = {} + + for team in teams: + teams_data[team.name] = { + "permission": team.permission, + "users": [ + member.login + for member in api.teams.list_members_in_org(GITHUB_ORG, team.slug) + ], + } + + # "direct" here means we don't list each individual member of teams that + # also have access to the repo: + users = api.repos.list_collaborators(repo.name, affiliation="direct") + + for user in sorted(users, key=lambda u: u.login): + for team, data in teams_data.items(): + if user.login not in data["users"]: + continue + # This repo gives access to a team that already includes the + # explicitly listed user. See if the user is added with + # greater permissions, otherwise this user is redundant. + team_access = get_access_level(data["permission"]) + user_access = get_access_level(user.permissions) + + if user_access <= team_access: + result.append( + { + "repo": repo.name, + "user": user.login, + "redundancy": team, + } + ) + + print(json.dumps(result, sort_keys=True)) + return True + + +def cmd_repos_remove_redundant_users(api, args): + result = [] + outcome = True + + for repo in get_all_repos(api): + teams = api.repos.list_teams(repo.name) + teams_data = {} + + for team in teams: + teams_data[team.name] = { + "permission": team.permission, + "users": [ + member.login + for member in api.teams.list_members_in_org(GITHUB_ORG, team.slug) + ], + } + + # "direct" here means we don't list each individual member of teams that + # also have access to the repo: + users = api.repos.list_collaborators(repo.name, affiliation="direct") + + for user in sorted(users, key=lambda u: u.login): + for team, data in teams_data.items(): + if user.login not in data["users"]: + continue + # This repo gives access to a team that already includes the + # explicitly listed user. See if the user is added with + # greater permissions, otherwise this user is redundant. + team_access = get_access_level(data["permission"]) + user_access = get_access_level(user.permissions) + + if user_access <= team_access: + try: + if not args.dry_run: + api.repos.remove_collaborator(repo.name, user.login) + result.append( + { + "repo": repo.name, + "user": user.login, + "success": True, + } + ) + except HTTPError as err: + result.append( + { + "repo": repo.name, + "user": user.login, + "success": False, + "code": err.code, + "reason": err.reason, + } + ) + outcome = False + + print(json.dumps(result, sort_keys=True)) + return outcome + + +def cmd_repos_add_team(api, args): + if args.access_level not in [level.name.lower() for level in AccessLevel]: + print_err( + f"The given access level '{args.access_level}' is not valid. See --help.\n" + ) + return False + + access_level = access_level_to_github(AccessLevel[args.access_level.upper()]) + + # Resolve the given team name to a slug: + team_slug = None + teams = sorted(api.teams.list(GITHUB_ORG), key=lambda t: t.name) + for team in teams: + if ( + team.name.lower() == args.team.lower() + or team.slug.lower() == args.team.lower() + ): + team_slug = team.slug + break + + if not team_slug: + print_err( + f"Team '{args.team}' is not valid for the '{GITHUB_ORG}' organization." + ) + print_err(f"Must be one of {', '.join(["'" + t.name + "'" for t in teams])}.") + return False + + result = [] + outcome = True + + for repo in get_all_repos(api): + try: + if not args.dry_run: + api.teams.add_or_update_repo_permissions_in_org( + GITHUB_ORG, team_slug, repo.name, access_level + ) + result.append( + { + "repo": repo.name, + "success": True, + } + ) + except HTTPError as err: + result.append( + { + "repo": repo.name, + "success": False, + "code": err.code, + "reason": err.reason, + } + ) + outcome = False + + print(json.dumps(result, sort_keys=True)) + return outcome + + +def main(): + top_parser = argparse.ArgumentParser( + description="Helpers for automated Zeek repo administration on Github." + ) + + top_parser.add_argument( + "--auth-token", + default=None, + help="Your Github API token. You can also use the GITHUB_TOKEN environment " + "variable to provide this. As a fallback, this script tries to use your " + "gh configuration, if available.", + ) + + cmd_parser = top_parser.add_subparsers( + title="commands", + dest="command", + help="See `%(prog)s -h` for per-command usage info.", + ) + + repos_parser = cmd_parser.add_parser( + "repos", help="Do something for every repo in the Zeek organization." + ) + + repos_cmd_parser = repos_parser.add_subparsers( + title="commands", + dest="command", + help="See `%(prog)s -h` for per-command usage info.", + ) + + repos_list_access_cmd_parser = repos_cmd_parser.add_parser( + "list-access", + help="List all users and teams that have access to a repo, along with the access level.", + ) + repos_list_access_cmd_parser.set_defaults(run_cmd=cmd_repos_list_access) + + repos_list_redundant_users_cmd_parser = repos_cmd_parser.add_parser( + "list-redundant-users", + help="List users already covered via teams providing at least the same access level.", + ) + repos_list_redundant_users_cmd_parser.set_defaults( + run_cmd=cmd_repos_list_redundant_users + ) + + repos_remove_redundant_user_cmd_parser = repos_cmd_parser.add_parser( + "remove-redundant-users", + help="Remove users already included via a team that provides at least the same access level.", + ) + repos_remove_redundant_user_cmd_parser.set_defaults( + run_cmd=cmd_repos_remove_redundant_users + ) + repos_remove_redundant_user_cmd_parser.add_argument( + "--dry-run", + action="store_true", + help="Don't make any changes, just pretend API calls succeed", + ) + + repos_add_team_cmd_parser = repos_cmd_parser.add_parser( + "add-team", + help="Add the given team to each repository at the given access level. Teams that already have access are updated to the provided level.", + ) + repos_add_team_cmd_parser.set_defaults(run_cmd=cmd_repos_add_team) + + repos_add_team_cmd_parser.add_argument( + "--dry-run", + action="store_true", + help="Don't make any changes, just pretend API calls succeed", + ) + repos_add_team_cmd_parser.add_argument( + "--team", required=True, help="The name or slug of the team. Case-insensitive." + ) + repos_add_team_cmd_parser.add_argument( + "--access-level", + required=True, + help=f"The access level. One of {', '.join(["'" + level.name.lower() + "'" for level in AccessLevel])}.", + ) + + if "argcomplete" in sys.modules: + argcomplete.autocomplete(top_parser) + + args = top_parser.parse_args() + api = GhApi(owner=GITHUB_ORG, token=get_api_token(args)) + + try: + return args.run_cmd(api, args) + except HTTPError as err: + sys.stderr.write(f"API error at {err.url}: {err.code}, {err.reason}\n") + return False + except KeyboardInterrupt: + return False + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/auxil/zeek-aux/devel-tools/make-release b/auxil/zeek-aux/devel-tools/make-release new file mode 100755 index 0000000000..38a4232921 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/make-release @@ -0,0 +1,129 @@ +#! /usr/bin/env bash +# +# Makes release tarballs for the repository in the current directory; +# and if --recursive is given, also for all submodules. + +DEST="$(pwd)/build/dist" # Where to put the TGZs. +LOGS="$DEST/.log" # Where to put build logs. + +dists="" + +function usage() { + echo "Usage: $(basename $0) [--recursive]" +} + +function release() { + mod=$1 + tmp="$LOGS/make-release.$(basename $mod).log" + + cat $tmp + + echo "--- Building distribution for $mod" >&2 + echo "Log in $tmp." >&2 + + make dist >$tmp 2>&1 + + cat $tmp | awk '/CPack:.*generated/ {print $4} /^Package: / {print $2}' | while read tgz; do + if echo $tgz | grep -qv ^/; then + tgz="$(pwd)/$tgz" + fi + + version=$(git tag --contains HEAD | grep -E '^(release|beta)$') + + if [ "$version" == "" ]; then + version="git" + fi + + echo "Distribution in $tgz ($version)" >&2 + ls -al $tgz | awk '{print " ", $0; }' >&2 + + echo "$tgz#$version" + done +} + +if [ "$1" == "--recursive" ]; then + submodules=1 + shift +fi + +if [ "$submodules" == "1" ]; then + mods=$(git submodule foreach -q --recursive pwd | grep -v /cmake) +fi + +if [ -e $DEST ]; then + echo + echo "$DEST exists already, proceeding will delete it." + echo ">> Continue? (y/n)" + read line + if [ "$line" != "y" ]; then + exit 1 + fi +fi + +rm -rf $DEST +install -d $DEST +install -d $LOGS + +mods="$mods ." + +build= + +for mod in $mods; do + cwd=$(pwd) + cd $mod + + if [ "$mod" = "." ]; then + mod=$(pwd) + mod=$(basename $mod) + fi + + if [ ! '(' -e CMakeLists.txt -o -e Makefile ')' ]; then + echo "No CMakeLists.txt or Makefile in $mod, skipping." + cd $cwd + continue + fi + + ignore=1 + + if [ "$(git describe HEAD)" = "$(git describe release 2>/dev/null)" ]; then + ignore=0 + fi + + if [ "$(git describe HEAD)" = "$(git describe beta 2>/dev/null)" ]; then + ignore=0 + fi + + if [ "$ignore" = "1" ]; then + echo "Module $mod is not tagged for release or beta, skipping." + cd $cwd + continue + fi + + dist=$(release $mod) + (echo $dist | grep -qv "^ *$") || echo "No distribution found for $mod." + + dists="$dists $dist" + echo + + cd $cwd +done + +for dist in $dists; do + tgz=$(echo $dist | cut -d '#' -f 1) + version=$(echo $dist | cut -d '#' -f 2) + dst=$(basename $tgz) + dst="$DEST/$version/$(echo $dst | sed 's/\.tgz/.tar.gz/g')" + + install -d $DEST/$version + mv $tgz $dst + + echo Signing $dst ... + sign-file $dst + +done + +echo "--- All distributions in $DEST:" + +all=$(find $DEST -path '*gz*') +test "$all" != "" && ls -rl $all || echo "None." +echo diff --git a/auxil/zeek-aux/devel-tools/perf-benchmark b/auxil/zeek-aux/devel-tools/perf-benchmark new file mode 100755 index 0000000000..bffab695f8 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/perf-benchmark @@ -0,0 +1,230 @@ +#!/bin/sh + +ZEEK_BUILD="" +DATA_FILE="" +MODE="intf" +INTERFACE="" +SEED_FILE="" + +# Path where flamegraph is installed +FLAMEGRAPH_PATH="" +FLAMEGRAPH_PREFIX="benchmark" + +usage() { + usage="\ +Usage: $0 -z [zeek binary path] -d [data file path] + + Options: + -b, --build PATH The path to a Zeek binary to benchmark + -d, --data-file PATH The path to a data file to read from for replay + -m, --mode MODE This can be one of three possible values: + intf, read, or flamegraph. This controls what + mode is used for the benchmark run, and defaults + to intf if not passed. The modes are described + below. + -i, --interface INTF The network interface to use for capturing data. + This interface should be completely idle, since + tcpreplay will be using it to replay the data. + This argument is ignored if the mode is 'file'. + -f, --flamegraph PATH The path to the directory where Flamegraph is + installed. This argument is required if the mode + is 'flamegraph', but is ignored otherwise. + -o, --output FILE The file prefix to use as output for Flamegraph. + This defaults to 'benchmark'. This argument is ignored + if the mode is not 'flamegraph'. + -s, --seed FILE (optional) A path to a Zeek random seed file. + This is used control the generation of connection + IDs and other data so it is consistent between + benchmarking runs. + + By default or when 'intf' is passed for the mode argument, the output will + include CPU, memory, etc statistics from Zeek processing all of the data + in the data file as if it was reading it live from the network. This mode + requires an interface to be passed using the -i argument. + + When 'file' is passed for the mode (-m) argument, the output will include + the runtime and maximum memory usage of Zeek when reading the data file + directly from disk. + + When 'flamegraph' is passed for the mode (-m) argument, this script will + output two flamegraphs for the process runtime in svg format. The first + flamegraph is a standard graph showing the time spent in functions, + stacked in the normal manner. The second graph is 'stack-reversed'. + + Symbols in Flamegraph outputs may not correctly stack unless the various + libraries linked into Zeek are built with frame pointers. This includes + glibc, libpcap, and openssl. Rebuilding those libraries with the + -fno-omit-frame-pointer compiler flag may provide more accurate output. + You can set libraries that get preloaded by setting the PRELOAD_LIBS + variable in the script. + + This script assumes that it is being run on a system with a large number + of CPU cores. If being used on a smaller system, modify this script and + set the ZEEK_CPU and TCPREPLAY_CPU variables to smaller values. +" + + echo "${usage}" + exit 1 +} + +while ( ("$#")); do + case "$1" in + -d | --data-file) + DATA_FILE=$2 + shift 2 + ;; + -b | --build) + ZEEK_BUILD=$2 + shift 2 + ;; + -m | --mode) + MODE=$2 + shift 2 + ;; + -i | --interface) + INTERFACE=$2 + shift 2 + ;; + -f | --flamegraph) + FLAMEGRAPH_PATH=$2 + shift 2 + ;; + -o | --output) + FLAMEGRAPH_PREFIX=$2 + shift 2 + ;; + -s | --seed) + SEED_FILE=$2 + shift 2 + ;; + esac +done + +if [ "${MODE}" != "intf" -a "${MODE}" != "file" -a "${MODE}" != "flamegraph" ]; then + echo "Error: -m argument should be one of 'intf', 'file', or 'flamegraph'" + echo + usage +fi + +if [ -z "${ZEEK_BUILD}" ]; then + echo "Error: -b argument is required and should point at a Zeek binary" + echo + usage +fi + +if [ -z "${DATA_FILE}" ]; then + echo "Error: -d argument is required and should point at a pcap file to replay" + echo + usage +fi + +if [ "${MODE}" != "file" -a -z "${INTERFACE}" ]; then + echo "Error: -i argument is required for the ${MODE} mode and should point to an idle network interface" + echo + usage +fi + +# Various run-time options +ZEEK_CPU=10 +TCPREPLAY_CPU=11 +PRELOAD_LIBS="" + +ZEEK_ARGS="" +if [ "${MODE}" != "file" ]; then + ZEEK_ARGS="-i af_packet::${INTERFACE}" +fi + +if [ -n "${SEED_FILE}" ]; then + ZEEK_ARGS="${ZEEK_ARGS} -G ${SEED_FILE}" +fi + +if [ "${MODE}" = "intf" ]; then + + TIME_FILE=$(mktemp) + + echo "####### Testing reading data file from a network interface #######" + echo "Running '${ZEEK_BUILD} ${ZEEK_ARGS}' against ${DATA_FILE}" + # Start zeek, find it's PID, then wait 10s to let it reach a steady state + taskset --cpu-list $ZEEK_CPU time -f "%M" -o $TIME_FILE $ZEEK_BUILD $ZEEK_ARGS & + TIME_PID=$! + + sleep 5 + ZEEK_PID=$(ps -ef | awk -v timepid="${TIME_PID}" '{ if ($3 == timepid) { print $2 } }') + renice -20 -p $ZEEK_PID >/dev/null + sleep 5 + echo "Zeek running on PID ${ZEEK_PID}" + + # Start perf stat on the zeek process + perf stat -p $ZEEK_PID & + PERF_PID=$! + + # Start replaying the data + echo "Starting replay" + taskset --cpu-list $TCPREPLAY_CPU tcpreplay -i $INTERFACE -q $DATA_FILE + + # Capture the average CPU usage of the process + CPU_USAGE=$(ps -p $ZEEK_PID -o %cpu=) + + # Kill everything + echo + kill -2 $ZEEK_PID + wait $TIME_PID + wait $PERF_PID + + echo "Maximum memory usage (max_rss): $(head -n 1 ${TIME_FILE}) bytes" + echo "Average CPU usage: ${CPU_USAGE}%" + + rm $TIME_FILE + +elif [ "${MODE}" = "file" ]; then + + TIME_FILE=$(mktemp) + + echo "####### Testing reading the file directly from disk #######" + taskset --cpu-list $ZEEK_CPU time -f "%e %M" -o $TIME_FILE $ZEEK_BUILD $ZEEK_ARGS -r $DATA_FILE + TIME_PID=$! + ZEEK_PID=$(ps -ef | awk -v timepid="${TIME_PID}" '{ if ($3 == timepid) { print $2 } }') + renice -20 -p $ZEEK_PID >/dev/null + awk '{print "Time spent: " $1 " seconds\nMax memory usage: " $2 " bytes"}' $TIME_FILE + + rm $TIME_FILE + +elif [ "${MODE}" = "flamegraph" ]; then + + echo "####### Generating flamegraph data #######" + + PERF_RECORD_FILE=$(mktemp) + PERF_COLLAPSED_FILE=$(mktemp) + + # Start zeek under perf record, then sleep for a few seconds to let it actually start up. For runs with + # shorter amounts of data or with slower traffic, you can add '-c 499' here to get finer-grained results. + # With big data sets, it just results in the graph getting blown out by waits in the IO loop. + LD_PRELOAD=${PRELOAD_LIBS} perf record -g -o $PERF_RECORD_FILE -- $ZEEK_BUILD $ZEEK_ARGS & + PERF_PID=$! + + sleep 5 + + ZEEK_PID=$(ps -ef | awk -v perfpid="${PERF_PID}" '{ if ($3 == perfpid) { print $2 } }') + echo "Zeek running on PID ${ZEEK_PID}" + + # Start replaying the data + echo "Starting replay" + taskset --cpu-list $TCPREPLAY_CPU tcpreplay -i $INTERFACE -q $DATA_FILE + + # Kill everything + echo + kill -2 $ZEEK_PID + wait $PERF_PID + + echo + echo "####### Collapsing perf stack data #######" + perf script -i $PERF_RECORD_FILE | ${FLAMEGRAPH_PATH}/stackcollapse-perf.pl >$PERF_COLLAPSED_FILE + echo "####### Building normal flamegraph, writing to ${FLAMEGRAPH_PREFIX}.svg #######" + cat $PERF_COLLAPSED_FILE | ${FLAMEGRAPH_PATH}/flamegraph.pl >"${FLAMEGRAPH_PREFIX}.svg" + echo "####### Building reverse flamegraph, writing to ${FLAMEGRAPH_PREFIX}-reversed.svg #######" + cat $PERF_COLLAPSED_FILE | ${FLAMEGRAPH_PATH}/flamegraph.pl --reverse >"${FLAMEGRAPH_PREFIX}-reversed.svg" + + rm $PERF_RECORD_FILE + rm $PERF_COLLAPSED_FILE + +fi diff --git a/auxil/zeek-aux/devel-tools/repo-status b/auxil/zeek-aux/devel-tools/repo-status new file mode 100755 index 0000000000..bd97b0f486 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/repo-status @@ -0,0 +1,18 @@ +#! /usr/bin/env bash +# +# Helper for check-release. + +repo=$(basename $(git config --get remote.origin.url) | sed 's/^[^:]*://') +head=$(basename $(git symbolic-ref -q HEAD)) + +update_changes="$(dirname $0)/update-changes" + +printf "%20s " "$repo" +printf "%-10s " "$head" +printf "%-8s " $($update_changes -c | grep -q NOT && echo 'old!' || echo ok) +printf "%-8s " $(git log --oneline $head ^origin/$head | wc -l | awk '$1==0{printf("%s",$1)} $1!=0{printf("%s!",$1)}') +printf "%-7s " $(git status --porcelain | awk 'NF!=1' | grep -q '.' && echo 'Mod!' || echo ok) +printf "%-5s " $(git submodule status --recursive | sed 's/^\(\.\).*/\1/g' | grep -q '^ $' && echo 'Mod!' || echo ok) +printf "%-15s " $(cat VERSION | awk '/-/{printf("%s!", $1); next} {printf("%s", $1)}') +printf "%s" $(git tag -l --contains HEAD | sort -r | paste -sd ',' | awk '/release|beta/{printf("%s",$1); next;} {printf("%s!",$1)}') +echo diff --git a/auxil/zeek-aux/devel-tools/sign-file b/auxil/zeek-aux/devel-tools/sign-file new file mode 100755 index 0000000000..36c2f9c804 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/sign-file @@ -0,0 +1,29 @@ +#! /usr/bin/env bash +# +# Signs $1 with the Zeek GPG key. Signature will be written to $1.asc. +# +# GPG agent needs to be running. +# +# Needs gpg2. + +GPG=$(which gpg2) +KEY=F8CB8019 + +if [ "$GPG" == "" ]; then + echo "Can't find gpg2." + exit 1 +fi + +if [ "$#" != 1 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +if ! gpg-agent -q; then + echo "GPG agent not running." + exit 1 +fi + +FILE=$1 + +$GPG --detach-sign -a -u $KEY --openpgp -o $FILE.asc $FILE diff --git a/auxil/zeek-aux/devel-tools/update-changes b/auxil/zeek-aux/devel-tools/update-changes new file mode 100755 index 0000000000..1eb7d8e936 --- /dev/null +++ b/auxil/zeek-aux/devel-tools/update-changes @@ -0,0 +1,630 @@ +#! /usr/bin/env bash +# +# Assembles a draft CHANGES entry out of revisions committed since the last +# entry was added. The entry is prepended to the current CHANGES file, and the +# user then gets a chance to further edit it in the editor before it gets +# committed. +# +# The script also maintains and updates a VERSION file. +# +# If the script finds a file called .update-changes.cfg it sources it at the +# beginning. That script can define a function "new_version_hook" that will be +# called with the new version number. It may use any of the replace_version_* +# functions defined below to update other files as necessary. +# +# If $1 is given, it's interpreted as a release version and a corresponding +# tag is created. +# +# To start using update-changes in a new project, proceed as follows: +# +# (1) Run "update-changes -I". This will initialize the CHANGES file and, if +# needed, establish suitable git tags that update-changes requires in order +# to start enumerating commits after a release. You can also prepare the +# initial version number in the VERSION file if you prefer that approach. +# +# (2) If you're planning to use an .update-changes.cfg file, add it as well as +# any corresponding changes it requires. Continue regular development, and +# when ready, run update-changes to reflect the first actual changeset in +# the CHANGES file. +# +file_changes="CHANGES" # The CHANGES file. +file_version="VERSION" # The VERSION file. +file_config=".update-changes.cfg" # This will be sourced if available. +new_version_hook="new_version_hook" # Function that will be called with new version number. +new_commit_msg="Updating CHANGES and VERSION." # Commit message when creating a new commit. +init_commit_msg="Starting CHANGES." # Commit message when we initialize CHANGES +show_authors=1 # Include author names with commit. + +# The command line used to generate a revision's version string, such as +# v1.0.0-23-gabcdef. This relies on tags to work, which update-changes checks +# for. By default this only finds annotated tags; to allow lightweight ones as +# well, add --tags. +git_describe="git describe --tags" # {rev} will be added. + +# The command line used to generate a revision's date. The revision will be +# appended. Not used with Zeek-style CHANGES file. +git_rev_date="git show -s --pretty=tformat:%ci" + +# The command line used to generate the list of revisions between old and new +# state. +git_rev_list="git rev-list --topo-order HEAD" # ^{past-rev} will be added. + +# The command line used to show the one-line summary of a revision before +# editing. +git_rev_summary="git show -s '--pretty=tformat: %h | %aN | %s'" # {rev} will be added. + +# The command line used to get a revision's author. +git_author="git show -s --pretty=format:%aN" # {rev} will be added. +git_author_email="git show -s --pretty=format:%aE" # {rev} will be added. + +# The command line used to get a revision's message. +git_msg=default_format_msg # {rev} will be added. + +# Portable access to ERE, see e.g. https://unix.stackexchange.com/a/131940 +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +function usage { + echo "usage: $(basename $0) [options]" + echo + echo " -p Explicitly name the past revision to compare with." + echo " -R Tag the current revision as a release. Update VERSION to use that." + echo " -B Tag the current revision as a beta release. Update VERSION to use that." + echo " -r Tag the current revision as a release, using the next point version as version tag." + echo " -I Initialize a new, initially empty CHANGES file." + echo " -c Check whether CHANGES is up to date." + echo " -n Do not amend the HEAD commit when feasible, create a new one." + echo + exit 1 +} + +# Takes a version string as input and turns it into a Python-styled one. For +# example, input "1.2-23" becomes "1.2.dev23". Other formats remain +# unchanged. See: https://peps.python.org/pep-0440/#version-scheme +function pythonic_version { + echo "$1" | $sed "s#-#.dev#" +} + +### Functions that can be used to replace version strings in other files. +### To use them, create a file $file_config and define a function +### "new_version_hook" in there that does whatever is necessary, like calling +### any of these. + +# Function that looks for lines of the form 'VERSION="1.2.3"' in $1. It will +# replace the version number with $2 and then git-adds the change. +function replace_version_in_script { + file=$1 + version=$2 + + cat $file | $sed "s#^([[:blank:]]*VERSION[[:blank:]]*=[[:blank:]]*)\"([0-9.-]+)\"#\1\"$version\"#g" >$file.tmp + cat $file.tmp >$file + rm -f $file.tmp + git add $file +} + +# Function that looks for lines of the form '.. |version| replace:: 0.3' in $1. +# It will replace the version number with $2 and then git-adds the change. +function replace_version_in_rst { + file=$1 + version=$2 + + cat $file | $sed "s#^([[:blank:]]*\.\.[[:blank:]]*\|version\|[[:blank:]]*replace::[[:blank:]]*)([0-9a-zA-Z.-]+)#\1$version#g" >$file.tmp + cat $file.tmp >$file + rm -f $file.tmp + git add $file +} + +# Function that checks file $1 for lines starting with 'version = +# ""', where the version string can be of release form (e.g. "1.2.3") +# or a development one, expressed as "1.2-23" or the Python-styled "1.2.dev23". +# It will replace the version number with a Python-styled form of $2, then +# git-add the change. +function replace_version_in_setup_py { + file=$1 + version=$(pythonic_version $2) + + # The version string can be a sequence of digits and dots, optionally + # followed by either "-" or ".dev" plus at least one digit. + cat $file | $sed "s#^([[:blank:]]*version[[:blank:]]*=[[:blank:]]*)\"[0-9.]+((-|\.dev)[0-9]+)?\"#\1\"$version\"#g" >$file.tmp + cat $file.tmp >$file + rm -f $file.tmp + git add $file +} + +# Function that checks file $1 for lines starting with '__version__ = +# ""', where the version string can be of release form (e.g. "1.2.3") +# or a development one, expressed as "1.2-23" or the Python-styled "1.2.dev23". +# It will replace the version number with a Python-styled form of $2, then +# git-add the change. +function replace_version_in_python_package { + file=$1 + version=$(pythonic_version $2) + + # The version string can be a sequence of digits and dots, optionally + # followed by either "-" or ".dev" plus at least one digit. + cat $file | $sed "s#^([[:blank:]]*__version__[[:blank:]]*=[[:blank:]]*)\"[0-9.]+((-|\.dev)[0-9]+)?\"#\1\"$version\"#g" >$file.tmp + cat $file.tmp >$file + rm -f $file.tmp + git add $file +} + +# Function that looks for lines of the form "#define .*VERSION "0.3"", with the +# number being "version * 100". It will replace the version with $2 and then +# git-adds the change. +function replace_version_in_c_header { + file=$1 + version=$2 + + cat $file | $sed "s#([[:blank:]]*\#define[[:blank:]]*[_A-Za-z0-9]*_VERSION[[:blank:]]*)\"[0-9.-]+\"#\1\"$version\"#g" >$file.tmp + mv $file.tmp $file + git add $file +} + +# Default function for preparing commit message. This scans the message for +# GitHub issue references to include. +function default_format_msg { + if command -v gawk &>/dev/null; then + # We need gawk for the match(). + git show -s --pretty=format:%B $1 | + gawk ' + match($0, "([Ii]ssue|[Gg][Hh]|#)[ _-]?([0-9]+)", x) { issues[x[2]] = 1; } + { msg = msg $0 "\n"; } + END { + if ( ! match(msg, "^GH-[0-9]+") ) { + for ( i in issues ) + printf("GH-%s: ", i); + } + + print msg; + }' + else + git show -s --pretty=format:%B $1 + fi +} + +### + +function version { + rev=$1 + $git_describe $rev --match "v*" 2>/dev/null | $sed 's/^v//g' | $sed 's/-g.*//g' | $sed 's/-([[:alnum:]]+)-([0-9]+)$/-\1.\2/g' +} + +function start_changes_entry { + version=$1 + dst=$2 + + if [ "$zeek_style" == "0" ]; then + date=$($git_rev_date HEAD) + printf '%s | %s\n' "$version" "$date" >>$dst + else + date=$(date) + printf '%s %s\n' "$version" "$date" >>$dst + fi +} + +function add_to_changes_entry { + rev=$1 + dst=$2 + msg=$3 + + author="" + + if [ "$msg" == "" ]; then + if [ "$show_authors" == "1" ]; then + author_email=$($git_author_email $rev) + author=$($git_author $rev) + + if [[ "$author_email" == *@corelight.com ]]; then + author=" ($author, Corelight)" + else + author=" ($author)" + fi + fi + + msg=$($git_msg $rev) + fi + + if [ "$msg" == "" ]; then + return 1 + fi + + if echo $msg | grep -q "^$new_commit_msg\$"; then + # Ignore our own automated commits. + return 1 + fi + + if [[ $(git show --no-patch --format='%P' "$rev" | wc -w) -gt 1 ]]; then + # Ignore merge commits, i.e., commits with more than one parent. + return 1 + fi + + echo >>$dst + + if [ "$zeek_style" == "0" ]; then + bullet=" *" + else + bullet="-" + fi + + echo -n "$msg" | + awk -v bullet="$bullet" -v author="$author" 'NR==1{printf "%s %s%s\n", bullet, $0, author; next }{printf " %s\n", $0}' | + $sed 's/[[:blank:]]*$//' >>$dst + + return 0 +} + +function init_changes { + for rev in $(git rev-list HEAD); do + version=$(version $rev) + [ -n "$version" ] && break + done + + git_version=$version + + if [ "$version" == "" ] && [ -f $file_version ]; then + # git doesn't offer version info, but there's a VERSION file. + # Consider it if the user's okay with it. + version=$(cat $file_version | head -1) + if [ -n "$version" ]; then + echo "This git repo doesn't yet offer suitable version tags." + read -p "Use '$version' from $file_version? Y/n " -n 1 -r + echo + if [ -n "$REPLY" ] && [[ $REPLY != [Yy] ]]; then + version= + fi + fi + fi + + if [ "$version" == "" ]; then + read -p "No initial version available, please provide one (e.g. 0.1, 1.0.0): " -r version + if [[ "$version" == v* ]]; then + # We don't need a "v" prefix here, it only exists in the git tags. + version=${version:1} + fi + fi + + # Subtle: if we're basing this CHANGES intro on a git tag, then we have a + # chicken-and-egg problem with correct numbering of the commit introducing + # CHANGES. It is itself going to be the _next_ commit. We could increment + # the git-derived version number arithmetically, but it's easier to just + # commit an empty CHANGES and then augment that below. + if [ -n "$git_version" ]; then + touch $file_changes + git add $file_changes + git commit -m "$init_commit_msg" + version=$(version HEAD) + flags="--amend" + fi + + start_changes_entry $version $file_changes + echo >>$file_changes + echo " * Starting $file_changes." >>$file_changes + + git add $file_changes + git commit $flags -m "$init_commit_msg" + + if [ -z "$git_version" ]; then + git tag "v$version" + fi +} + +function get_last_rev { + version=$(cat $file_changes | grep -E '^[0-9a-zA-Z.-]+ *\|' | head -1 | awk '{print $1}') + + if echo $version | grep -q -- '-'; then + # version is now e.g. 1.0.4-14 -- find the revision with that number. + for rev in $(git rev-list HEAD); do + v=$(version $rev) + + if [ "$v" == "$version" ]; then + echo $rev + return + fi + done + + echo "Cannot determine revision for version $version." >/dev/stderr + exit 1 + + else + # A tag. + echo "v$version" + fi +} + +function check_release_tag { + if [ "$release" != "" ]; then + git tag -d $release 2>/dev/null + git tag -a $release -m "Version tag" + echo "Tagged with new tag $release." + echo + echo "Push with: " + echo + echo " git push origin && git push origin $release" + echo + fi +} + +function check_beta_tag { + if [ "$beta" != "" ]; then + git tag -d $beta 2>/dev/null + git tag -a $beta -m "Beta version tag" + echo "Tagged with new tag $beta." + echo + echo "Push with: " + echo + echo " git push origin && git push origin $beta" + echo + fi +} + +function check_submodules { + if git submodule status --recursive | grep ^+; then + cat <".' + exit 1 +fi + +auto_version=$(version HEAD) + +if [ "$auto_version" == "" ]; then + echo "Cannot determine version, checking HEAD did not return anything." + exit 1 +fi + +tmp=${file_changes}.$$.tmp +trap "rm -f $tmp" EXIT +rm -f $tmp + +found=0 + +new_version=$auto_version +version=$(version $rev) + +if [ "$version" == "" ]; then + echo "Cannot determine version for $rev." + exit 1 +fi + +if [ "$release" != "" ]; then + new_version=$(echo $release | sed 's/v//g') +fi + +if [ "$beta" != "" ]; then + new_version=$(echo $beta | sed 's/v//g') +fi + +if [ "$quiet" != "1" ]; then + echo "New version is $new_version." + echo "Listing revisions committed since $(version $last_rev) ($last_rev) ... " + echo +fi + +start_changes_entry $new_version $tmp + +for rev in $($git_rev_list ^$last_rev); do + + version=$(version $rev) + + if [ "$version" == "" ]; then + version="" + fi + + # printf "%15s |" $version + + if add_to_changes_entry $rev $tmp; then + found=1 + + if [ "$quiet" != "1" ]; then + eval "$git_rev_summary $rev | grep -v '^$' | cat" + fi + fi + +done + +if [ "$found" == "0" ]; then + if [ "$check" == "1" ]; then + echo "CHANGES is up to date." + exit 0 + fi + + echo " None." + echo + + if [ "$release" != "" -o "$beta" != "" ]; then + add_to_changes_entry head $tmp "Release $new_version." + else + exit 0 + fi +fi + +if [ "$check" == "1" ]; then + echo "CHANGES is NOT up to date." + exit 1 +fi + +echo >>$tmp + +cat $file_changes >>$tmp + +# If we are ahead of origin, we can amend. If not, we need to create a new +# commit even if the user wants otherwise. If the user requested -n (no +# amendments), we skip all of this. +amend=0 + +if [ $no_amends == "0" ] && + git remote | grep -q origin && + git rev-list origin/$(git rev-parse --abbrev-ref HEAD)..HEAD | grep -q .; then + amend=1 +fi + +echo + +if [ "$amend" == "0" ]; then + echo Update to $file_changes will become a new commit. +else + echo Update to $file_changes will be amended to last commit. +fi + +echo +echo Type Enter to edit new $file_changes, or CTRL-C to abort without any modifications. +read + +# Run editor. +if [ -z "$EDITOR" ]; then + EDITOR=vi +fi +eval $EDITOR $tmp + +# Put changes in place. +mv $tmp $file_changes +echo "Updated $file_changes." + +if [ "$file_version" != "" ]; then + echo $new_version >$file_version + echo "Updated $version to $new_version." +fi + +# Call hook function if it exists. +if type $new_version_hook >/dev/null 2>&1; then + $new_version_hook $new_version +fi + +# Commit changes. +git add $file_changes $file_version + +if [ "$amend" == "1" ]; then + git commit --amend +else + git commit -m "$new_commit_msg" +fi + +echo "Updates committed." + +check_release_tag +check_beta_tag diff --git a/auxil/zeek-aux/devel-tools/zeek_lldb_utils.py b/auxil/zeek-aux/devel-tools/zeek_lldb_utils.py new file mode 100644 index 0000000000..31638b85ed --- /dev/null +++ b/auxil/zeek-aux/devel-tools/zeek_lldb_utils.py @@ -0,0 +1,57 @@ +import lldb +from ansi.color import fg +from ansi.color.fx import reset +from linereader import getline + +# Cache script lines and files so that we don't have to load files repeatedly +script_lines = {} + +# TODO: make this check that the thread is actually stopped and return an error if not + + +@lldb.command("btz") +def backtrace_zeek(debugger, command, exe_ctx, result, d): + selected_thread = exe_ctx.GetProcess().GetSelectedThread() + thread = exe_ctx.GetThread() + + # I'd prefer to retrieve this from LLDB somehow, but the earlier versions + # don't have SDDebugger.GetSetting(), and I'm not really sure we could use + # the output from that anyways. + thread_format = f"{'*' if selected_thread.idx == thread.idx else ' '} thread #{thread.idx}, name = '{thread.name}', queue = {fg.green}'{thread.queue}'{reset}, stop reason = {fg.red}{thread.GetStopDescription(100)}{reset}" + print(thread_format) + + selected_frame = thread.GetSelectedFrame().idx + + for frame in thread.get_thread_frames(): + frame_output = f" {'*' if frame.idx == selected_frame else ' '} " + frame_output += f"{frame}" + + this = frame.FindVariable("this") + if this: + loc_ptr = this.GetChildMemberWithName("location") + if loc_ptr and loc_ptr.GetType().GetName() == "zeek::detail::Location *": + if loc_ptr.GetValueAsUnsigned() != 0: + loc = frame.EvaluateExpression("*(this->location)") + fname = ( + loc.GetChildMemberWithName("filename").GetSummary().strip('"') + ) + line_no = loc.GetChildMemberWithName( + "first_line" + ).GetValueAsUnsigned() + frame_output += f"\n {fg.green}zeek script:{reset} {fname}" + + fileinfo = f"{fname}:{line_no}" + if fileinfo in script_lines: + line = script_lines[fileinfo] + else: + line = getline(fname, line_no) + line = line.strip() + script_lines[fileinfo] = line + + if line: + line_hdr = f"line {line_no}" + frame_output += ( + f"\n {fg.green}{line_hdr: >11}:{reset} {line}" + ) + + print(frame_output) diff --git a/auxil/zeek-aux/plugin-support/README b/auxil/zeek-aux/plugin-support/README new file mode 100644 index 0000000000..6aa351e37d --- /dev/null +++ b/auxil/zeek-aux/plugin-support/README @@ -0,0 +1,4 @@ +The init-plugin script instantiates a plugin template to provide you with a good +starting point for your next Zeek plugin. Use this only if you want to develop +a "pure" Zeek plugin -- if you're looking to start a Zeek package for the zkg +package manager, then use the "zkg create" command instead. diff --git a/auxil/zeek-aux/plugin-support/init-plugin b/auxil/zeek-aux/plugin-support/init-plugin new file mode 100755 index 0000000000..d26b49b469 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/init-plugin @@ -0,0 +1,96 @@ +#! /usr/bin/env bash + +function abspath { + ( + cd "$1" + pwd + ) +} + +function relpath { + echo "$1" | sed "s#$(pwd)/\{0,1\}##g" +} + +function lower { + echo "$1" | tr A-Z a-z +} + +function upper { + echo "$1" | tr a-z A-Z +} + +function init-skeleton { + for i in $(cd ${skeleton} && find * -type d); do + mkdir -p $(echo ${plugin_src}/$i) + done + + for i in $(cd ${skeleton} && find * .??* -type f); do + src=${skeleton}/$i + dst=$(echo ${plugin_src}/$i) + dst=$(echo $dst | sed "s#%NS_LOWER#${plugin_ns_lower}#g") + dst=$(echo $dst | sed "s#%NAME_LOWER#${plugin_name_lower}#g") + dst=$(echo $dst | sed "s#%NS#${plugin_ns}#g") + dst=$(echo $dst | sed "s#%NAME#${plugin_name}#g") + dst=$(echo $dst | sed "s#@#/#g") + + tmp=$dst.$$.tmp + + mkdir -p $(dirname ${dst}) + cp -p ${src} ${tmp} # Copy first to preserve permissions. + cat ${src} | + sed "s/@PLUGIN_NAME@/${plugin_name}/g" | + sed "s/@PLUGIN_NAMESPACE@/${plugin_ns}/g" | + sed "s/@PLUGIN_NAME_UPPER@/${plugin_name_upper}/g" | + sed "s/@PLUGIN_NAMESPACE_UPPER@/${plugin_ns_upper}/g" | + sed "s/@PLUGIN_NAME_LOWER@/${plugin_name_lower}/g" | + sed "s/@PLUGIN_NAMESPACE_LOWER@/${plugin_ns_lower}/g" \ + >${tmp} + + if [ -e ${dst} ]; then + cmp -s ${tmp} ${dst} || echo "$(relpath ${dst}) exists, not installing new version" + rm -f ${tmp} + continue + fi + + echo Installing $(relpath ${dst}) ... + mv ${tmp} ${dst} + + done +} + +update=0 + +if [ "$1" == "-u" ]; then + update=1 + shift +fi + +if [ $# != 3 ]; then + echo "Usage: $(basename $0) [-u] " + exit 1 +fi + +dstdir=$1 + +if [ -e "${dstdir}" -a ${update} != 1 ]; then + echo "error: ${dstdir} already exists, use -u if you want to update skeleton files in there." + exit 1 +fi + +mkdir -p ${dstdir} +dstdir=$(abspath ${dstdir}) + +basedir=$(dirname $0) +skeleton=$(abspath ${basedir})/skeleton +plugin_ns=$2 +plugin_name=$3 +plugin_ns_lower=$(lower $2) +plugin_name_lower=$(lower $3) +plugin_ns_upper=$(upper $2) +plugin_name_upper=$(upper $3) +plugin_src=${dstdir} +plugin_build=${dstdir}/dylib + +init-skeleton + +which git >/dev/null 2>&1 && (cd ${dstdir} && git init . && git add .) diff --git a/auxil/zeek-aux/plugin-support/skeleton/.gitignore b/auxil/zeek-aux/plugin-support/skeleton/.gitignore new file mode 100644 index 0000000000..ee35264121 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/.gitignore @@ -0,0 +1,3 @@ +build +*.log +.state diff --git a/auxil/zeek-aux/plugin-support/skeleton/CHANGES b/auxil/zeek-aux/plugin-support/skeleton/CHANGES new file mode 100644 index 0000000000..e69de29bb2 diff --git a/auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt b/auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt new file mode 100644 index 0000000000..e196b0351f --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) + +project(ZeekPlugin@PLUGIN_NAME@) + +include(ZeekPlugin) + +zeek_plugin_begin(@PLUGIN_NAMESPACE@ @PLUGIN_NAME@ ${ZEEK_PLUGIN_BEGIN_OPTS}) +zeek_plugin_cc(src/Plugin.cc) +zeek_plugin_bif(src/@PLUGIN_NAME_LOWER@.bif) +zeek_plugin_dist_files(README CHANGES COPYING VERSION) +zeek_plugin_scripts(scripts/__load__.zeek scripts/__preload__.zeek scripts/types.zeek + scripts/@PLUGIN_NAMESPACE@/@PLUGIN_NAME@/__load__.zeek) +zeek_plugin_end() + +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) + +if ("${PROJECT_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}") + # Allows building rpm/deb packages via "make package" in build dir. + include(ConfigurePackaging) + ConfigurePackaging(${VERSION}) +endif () diff --git a/auxil/zeek-aux/plugin-support/skeleton/COPYING.edit-me b/auxil/zeek-aux/plugin-support/skeleton/COPYING.edit-me new file mode 100644 index 0000000000..ae2821fbaa --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/COPYING.edit-me @@ -0,0 +1,35 @@ +### +### This is a BSD-style license. If you're happy with it, just edit +### the XXX parts below and remove this comment. Otherwise, put in +### your own license instead. +### + +Copyright (c) 2018 by + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +(1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +(2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +(3) Neither the name of , nor + the names of contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/auxil/zeek-aux/plugin-support/skeleton/Makefile b/auxil/zeek-aux/plugin-support/skeleton/Makefile new file mode 100644 index 0000000000..e96720ffea --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/Makefile @@ -0,0 +1,29 @@ +# +# Convenience Makefile providing a few common top-level targets. +# + +cmake_build_dir=build +arch=`uname -s | tr A-Z a-z`-`uname -m` + +all: build-it + +build-it: + @test -e $(cmake_build_dir)/config.status || ./configure + -@test -e $(cmake_build_dir)/CMakeCache.txt && \ + test $(cmake_build_dir)/CMakeCache.txt -ot `cat $(cmake_build_dir)/CMakeCache.txt | grep ZEEK_DIST | cut -d '=' -f 2`/build/CMakeCache.txt && \ + echo Updating stale CMake cache && \ + touch $(cmake_build_dir)/CMakeCache.txt + + ( cd $(cmake_build_dir) && make ) + +install: + ( cd $(cmake_build_dir) && make install ) + +clean: + ( cd $(cmake_build_dir) && make clean ) + +distclean: + rm -rf $(cmake_build_dir) + +test: + make -C tests diff --git a/auxil/zeek-aux/plugin-support/skeleton/README b/auxil/zeek-aux/plugin-support/skeleton/README new file mode 100644 index 0000000000..f18fd5bcdf --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/README @@ -0,0 +1,5 @@ + +@PLUGIN_NAMESPACE@::@PLUGIN_NAME@ +================================= + + diff --git a/auxil/zeek-aux/plugin-support/skeleton/VERSION b/auxil/zeek-aux/plugin-support/skeleton/VERSION new file mode 100644 index 0000000000..6e8bf73aa5 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/auxil/zeek-aux/plugin-support/skeleton/configure b/auxil/zeek-aux/plugin-support/skeleton/configure new file mode 100755 index 0000000000..a251f01228 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/configure @@ -0,0 +1,190 @@ +#!/bin/sh +# +# Wrapper for viewing/setting options that the plugin's CMake +# scripts will recognize. +# +# Don't edit this. Edit configure.plugin to add plugin-specific options. +# + +set -e +command="$0 $*" + +if [ -e $(dirname $0)/configure.plugin ]; then + # Include custom additions. + . $(dirname $0)/configure.plugin +fi + +usage() { + + cat 1>&2 </dev/null 2>&1; then + plugin_usage 1>&2 + fi + + echo + + exit 1 +} + +# Function to append a CMake cache entry definition to the +# CMakeCacheEntries variable +# $1 is the cache entry variable name +# $2 is the cache entry variable type +# $3 is the cache entry variable value +append_cache_entry() { + CMakeCacheEntries="$CMakeCacheEntries -D $1:$2=$3" +} + +# set defaults +builddir=build +zeekdist="" +installroot="default" +zeek_plugin_begin_opts="" +CMakeCacheEntries="" + +while [ $# -ne 0 ]; do + case "$1" in + -*=*) optarg=$(echo "$1" | sed 's/[-_a-zA-Z0-9]*=//') ;; + *) optarg= ;; + esac + + case "$1" in + --help | -h) + usage + ;; + + --cmake=*) + CMakeCommand=$optarg + ;; + + --zeek-dist=*) + zeekdist=$(cd $optarg && pwd) + ;; + + --install-root=*) + installroot=$optarg + ;; + + --with-binpac=*) + append_cache_entry BinPAC_ROOT_DIR PATH $optarg + binpac_root=$optarg + ;; + + --with-broker=*) + append_cache_entry BROKER_ROOT_DIR PATH $optarg + broker_root=$optarg + ;; + + --with-bifcl=*) + append_cache_entry BifCl_EXE PATH $optarg + ;; + + --enable-debug) + append_cache_entry BRO_PLUGIN_ENABLE_DEBUG BOOL true + ;; + + --disable-cpp-tests) + zeek_plugin_begin_opts="DISABLE_CPP_TESTS;$zeek_plugin_begin_opts" + ;; + + *) + if type plugin_option >/dev/null 2>&1; then + plugin_option $1 && shift && continue + fi + + echo "Invalid option '$1'. Try $0 --help to see available options." + exit 1 + ;; + esac + shift +done + +if [ -z "$CMakeCommand" ]; then + # prefer cmake3 over "regular" cmake (cmake == cmake2 on RHEL) + if command -v cmake3 >/dev/null 2>&1; then + CMakeCommand="cmake3" + elif command -v cmake >/dev/null 2>&1; then + CMakeCommand="cmake" + else + echo "This plugin requires CMake, please install it first." + echo "Then you may use this script to configure the CMake build." + echo "Note: pass --cmake=PATH to use cmake in non-standard locations." + exit 1 + fi +fi + +if [ -z "$zeekdist" ]; then + if type zeek-config >/dev/null 2>&1; then + zeek_config="zeek-config" + else + echo "Either 'zeek-config' must be in PATH or '--zeek-dist=' used" + exit 1 + fi + + append_cache_entry BRO_CONFIG_PREFIX PATH $(${zeek_config} --prefix) + append_cache_entry BRO_CONFIG_INCLUDE_DIR PATH $(${zeek_config} --include_dir) + append_cache_entry BRO_CONFIG_PLUGIN_DIR PATH $(${zeek_config} --plugin_dir) + append_cache_entry BRO_CONFIG_LIB_DIR PATH $(${zeek_config} --lib_dir) + append_cache_entry BRO_CONFIG_CMAKE_DIR PATH $(${zeek_config} --cmake_dir) + append_cache_entry CMAKE_MODULE_PATH PATH $(${zeek_config} --cmake_dir) + + build_type=$(${zeek_config} --build_type) + + if [ "$build_type" = "debug" ]; then + append_cache_entry BRO_PLUGIN_ENABLE_DEBUG BOOL true + fi + + if [ -z "$binpac_root" ]; then + append_cache_entry BinPAC_ROOT_DIR PATH $(${zeek_config} --binpac_root) + fi + + if [ -z "$broker_root" ]; then + append_cache_entry BROKER_ROOT_DIR PATH $(${zeek_config} --broker_root) + fi +else + if [ ! -e "$zeekdist/zeek-path-dev.in" ]; then + echo "$zeekdist does not appear to be a valid Zeek source tree." + exit 1 + fi + + append_cache_entry ZEEK_DIST PATH $zeekdist + append_cache_entry CMAKE_MODULE_PATH PATH $zeekdist/cmake +fi + +if [ "$installroot" != "default" ]; then + mkdir -p $installroot + append_cache_entry BRO_PLUGIN_INSTALL_ROOT PATH $installroot +fi + +if [ -n "$zeek_plugin_begin_opts" ]; then + append_cache_entry ZEEK_PLUGIN_BEGIN_OPTS STRING "$zeek_plugin_begin_opts" +fi + +if type plugin_addl >/dev/null 2>&1; then + plugin_addl +fi + +echo "Build Directory : $builddir" +echo "Zeek Source Directory : $zeekdist" + +mkdir -p $builddir +cd $builddir + +"$CMakeCommand" $CMakeCacheEntries .. + +echo "# This is the command used to configure this build" >config.status +echo $command >>config.status +chmod u+x config.status diff --git a/auxil/zeek-aux/plugin-support/skeleton/configure.plugin b/auxil/zeek-aux/plugin-support/skeleton/configure.plugin new file mode 100644 index 0000000000..2664f4123d --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/configure.plugin @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Hooks to add custom options to the configure script. +# + +plugin_usage() +{ + : # Do nothing +# cat </`. Include code here that +# should execute at that point. This is the most common entry point to +# your plugin's accompanying scripts. +# + +# @load ./bar + diff --git a/auxil/zeek-aux/plugin-support/skeleton/scripts/__load__.zeek b/auxil/zeek-aux/plugin-support/skeleton/scripts/__load__.zeek new file mode 100644 index 0000000000..bcfe0efda8 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/scripts/__load__.zeek @@ -0,0 +1,10 @@ +# +# This is loaded automatically at Zeek startup once the plugin gets activated +# and its BiF elements have become available. Include code here that should +# always execute unconditionally at that time. +# +# Note that often you may want your plugin's accompanying scripts not here, but +# in scripts///__load__.zeek. That's processed +# only on explicit `@load /`. +# + diff --git a/auxil/zeek-aux/plugin-support/skeleton/scripts/__preload__.zeek b/auxil/zeek-aux/plugin-support/skeleton/scripts/__preload__.zeek new file mode 100644 index 0000000000..ada4e28927 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/scripts/__preload__.zeek @@ -0,0 +1,11 @@ +# +# This is loaded automatically at Zeek startup once the plugin gets activated, +# but before any of the BiFs that the plugin defines become available. +# +# This is primarily for defining types that BiFs already depend on. If you +# need to do any other unconditional initialization, that should go into +# __load__.zeek instead. +# + +@load ./types + diff --git a/auxil/zeek-aux/plugin-support/skeleton/scripts/types.zeek b/auxil/zeek-aux/plugin-support/skeleton/scripts/types.zeek new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/scripts/types.zeek @@ -0,0 +1 @@ + diff --git a/auxil/zeek-aux/plugin-support/skeleton/src/%NAME_LOWER.bif b/auxil/zeek-aux/plugin-support/skeleton/src/%NAME_LOWER.bif new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/src/%NAME_LOWER.bif @@ -0,0 +1 @@ + diff --git a/auxil/zeek-aux/plugin-support/skeleton/src/Plugin.cc b/auxil/zeek-aux/plugin-support/skeleton/src/Plugin.cc new file mode 100644 index 0000000000..60d3b79bbf --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/src/Plugin.cc @@ -0,0 +1,17 @@ + +#include "Plugin.h" + +namespace plugin { namespace @PLUGIN_NAMESPACE@_@PLUGIN_NAME@ { Plugin plugin; } } + +using namespace plugin::@PLUGIN_NAMESPACE@_@PLUGIN_NAME@; + +zeek::plugin::Configuration Plugin::Configure() + { + zeek::plugin::Configuration config; + config.name = "@PLUGIN_NAMESPACE@::@PLUGIN_NAME@"; + config.description = ""; + config.version.major = 0; + config.version.minor = 1; + config.version.patch = 0; + return config; + } diff --git a/auxil/zeek-aux/plugin-support/skeleton/src/Plugin.h b/auxil/zeek-aux/plugin-support/skeleton/src/Plugin.h new file mode 100644 index 0000000000..ef71264e28 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/src/Plugin.h @@ -0,0 +1,19 @@ + +#pragma once + +#include + +namespace plugin { +namespace @PLUGIN_NAMESPACE@_@PLUGIN_NAME@ { + +class Plugin : public zeek::plugin::Plugin +{ +protected: + // Overridden from zeek::plugin::Plugin. + zeek::plugin::Configuration Configure() override; +}; + +extern Plugin plugin; + +} +} diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/%NAME_LOWER@show-plugin.zeek b/auxil/zeek-aux/plugin-support/skeleton/tests/%NAME_LOWER@show-plugin.zeek new file mode 100644 index 0000000000..b072067267 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/%NAME_LOWER@show-plugin.zeek @@ -0,0 +1,2 @@ +# @TEST-EXEC: zeek -NN @PLUGIN_NAMESPACE@::@PLUGIN_NAME@ |sed -e 's/version.*)/version)/g' >output +# @TEST-EXEC: btest-diff output diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/.gitignore b/auxil/zeek-aux/plugin-support/skeleton/tests/.gitignore new file mode 100644 index 0000000000..fc422ef224 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/.gitignore @@ -0,0 +1,2 @@ +.btest.failed.dat +.tmp diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/Baseline@%NAME_LOWER.show-plugin@output b/auxil/zeek-aux/plugin-support/skeleton/tests/Baseline@%NAME_LOWER.show-plugin@output new file mode 100644 index 0000000000..29b1aeb7fd --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/Baseline@%NAME_LOWER.show-plugin@output @@ -0,0 +1,2 @@ +@PLUGIN_NAMESPACE@::@PLUGIN_NAME@ - (dynamic, version) + diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/Makefile b/auxil/zeek-aux/plugin-support/skeleton/tests/Makefile new file mode 100644 index 0000000000..4cdedac75f --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/Makefile @@ -0,0 +1,3 @@ + +test: + @btest diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/Scripts/diff-remove-timestamps b/auxil/zeek-aux/plugin-support/skeleton/tests/Scripts/diff-remove-timestamps new file mode 100755 index 0000000000..01c0c10d9b --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/Scripts/diff-remove-timestamps @@ -0,0 +1,13 @@ +#! /usr/bin/env bash +# +# Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | + $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/Scripts/get-zeek-env b/auxil/zeek-aux/plugin-support/skeleton/tests/Scripts/get-zeek-env new file mode 100755 index 0000000000..318d290f53 --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/Scripts/get-zeek-env @@ -0,0 +1,36 @@ +#! /bin/sh +# +# BTest helper for getting values for Zeek-related environment variables. + +base=$(dirname $0) +zeek_dist=$(cat ${base}/../../build/CMakeCache.txt | grep ZEEK_DIST | cut -d = -f 2) + +if [ -n "${zeek_dist}" ]; then + if [ "$1" = "zeekpath" ]; then + ${zeek_dist}/build/zeek-path-dev + elif [ "$1" = "zeek_plugin_path" ]; then + (cd ${base}/../.. && pwd) + elif [ "$1" = "path" ]; then + echo ${zeek_dist}/build/src:${zeek_dist}/aux/btest:${zeek_dist}/auxil/btest:${base}/:${zeek_dist}/aux/zeek-cut:${zeek_dist}/auxil/zeek-cut:$PATH + else + echo "usage: $(basename $0) " >&2 + exit 1 + fi +else + # Use Zeek installation for testing. In this case zeek-config must be in PATH. + if ! which zeek-config >/dev/null; then + echo "zeek-config not found" >&2 + exit 1 + fi + + if [ "$1" = "zeekpath" ]; then + zeek-config --zeekpath + elif [ "$1" = "zeek_plugin_path" ]; then + (cd ${base}/../.. && pwd) + elif [ "$1" = "path" ]; then + echo ${PATH} + else + echo "usage: $(basename $0) " >&2 + exit 1 + fi +fi diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/btest.cfg b/auxil/zeek-aux/plugin-support/skeleton/tests/btest.cfg new file mode 100644 index 0000000000..cabdb2190a --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/btest.cfg @@ -0,0 +1,17 @@ +[btest] +TestDirs = @PLUGIN_NAME_LOWER@ +TmpDir = %(testbase)s/.tmp +BaselineDir = %(testbase)s/Baseline +IgnoreDirs = .svn CVS .tmp +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store + +[environment] +ZEEKPATH=`%(testbase)s/Scripts/get-zeek-env zeekpath` +ZEEK_PLUGIN_PATH=`%(testbase)s/Scripts/get-zeek-env zeek_plugin_path` +ZEEK_SEED_FILE=%(testbase)s/random.seed +PATH=`%(testbase)s/Scripts/get-zeek-env path` +TZ=UTC +LC_ALL=C +TRACES=%(testbase)s/Traces +TMPDIR=%(testbase)s/.tmp +TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-remove-timestamps diff --git a/auxil/zeek-aux/plugin-support/skeleton/tests/random.seed b/auxil/zeek-aux/plugin-support/skeleton/tests/random.seed new file mode 100644 index 0000000000..6956a2c19a --- /dev/null +++ b/auxil/zeek-aux/plugin-support/skeleton/tests/random.seed @@ -0,0 +1,21 @@ +2983378351 +1299727368 +0 +310447 +0 +1409073626 +3975311262 +34130240 +1450515018 +1466150520 +1342286698 +1193956778 +2188527278 +3361989254 +3912865238 +3596260151 +517973768 +1462428821 +0 +2278350848 +32767 diff --git a/auxil/zeek-aux/rst/CMakeLists.txt b/auxil/zeek-aux/rst/CMakeLists.txt new file mode 100644 index 0000000000..6859379ab9 --- /dev/null +++ b/auxil/zeek-aux/rst/CMakeLists.txt @@ -0,0 +1,5 @@ +set(rst_SRCS rst.c) + +add_executable(rst ${rst_SRCS}) + +AddAuxInstallTarget(rst) diff --git a/auxil/zeek-aux/rst/rst.c b/auxil/zeek-aux/rst/rst.c new file mode 100644 index 0000000000..183ca7e60b --- /dev/null +++ b/auxil/zeek-aux/rst/rst.c @@ -0,0 +1,408 @@ +/* Derived from traceroute, which has the following copyright: + * + * Copyright (c) 1999, 2002 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: (1) source code distributions + * retain the above copyright notice and this paragraph in its entirety, (2) + * distributions including binary code include the above copyright notice and + * this paragraph in its entirety in the documentation or other materials + * provided with the distribution, and (3) all advertising materials mentioning + * features or use of this software display the following acknowledgement: + * ``This product includes software developed by the University of California, + * Lawrence Berkeley Laboratory and its contributors.'' Neither the name of + * the University nor the names of its contributors may be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef lint +static const char copyright[] = + "@(#) Copyright (c) 1999, 2002\nThe Regents of the University of California. All rights reserved.\n"; +#endif + +/* need this due to linux's funny idea of a tcphdr */ +#if defined(__linux__) +#define _DEFAULT_SOURCE +#define _BSD_SOURCE /* Deprecated, but still needed by older Linux. */ +#endif + +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include "config.h" + +/* Forwards */ +void gripe(const char *, const char *); +void pgripe(const char *); +uint16_t in_cksum(register uint16_t *, register int); +int ones_complement_checksum(const void *, int, uint32_t); +int tcp_checksum(const struct ip *, const struct tcphdr *, int); +void send_pkt(int, struct in_addr, int, uint32_t, struct in_addr, + int, uint32_t, int, int, int, int, const char *); +void terminate(int, const char *, int, uint32_t, const char *, + int, uint32_t, int, int, int, int, const char *); +void usage(void); +int main(int, char **); + +const char *prog_name; + +void gripe(const char *fmt, const char *arg) +{ + fprintf(stderr, "%s: ", prog_name); + fprintf(stderr, fmt, arg); + fprintf(stderr, "\n"); +} + +void pgripe(const char *msg) +{ + fprintf(stderr, "%s: %s (%s)\n", prog_name, msg, strerror(errno)); + exit(1); +} + +/* + * Checksum routine for Internet Protocol family headers (C Version) + */ +uint16_t +in_cksum(register uint16_t *addr, register int len) +{ + register int nleft = len; + register uint16_t *w = addr; + register uint16_t answer; + register int sum = 0; + + /* + * Our algorithm is simple, using a 32 bit accumulator (sum), + * we add sequential 16 bit words to it, and at the end, fold + * back all the carry bits from the top 16 bits into the lower + * 16 bits. + */ + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + /* mop up an odd byte, if necessary */ + if (nleft == 1) + sum += *(u_char *)w; + + /* + * add back carry outs from top 16 bits to low 16 bits + */ + sum = (sum >> 16) + (sum & 0xffff); /* add hi 16 to low 16 */ + sum += (sum >> 16); /* add carry */ + answer = ~sum; /* truncate to 16 bits */ + return (answer); +} + +// - adapted from tcpdump +// Returns the ones-complement checksum of a chunk of b short-aligned bytes. +int ones_complement_checksum(const void *p, int b, uint32_t sum) +{ + const uint16_t *sp = (uint16_t *) p; // better be aligned! + + b /= 2; // convert to count of short's + + /* No need for endian conversions. */ + while ( --b >= 0 ) + sum += *sp++; + + while ( sum > 0xffff ) + sum = (sum & 0xffff) + (sum >> 16); + + return sum; +} + +int tcp_checksum(const struct ip *ip, const struct tcphdr *tp, int len) +{ + int tcp_len = tp->th_off * 4 + len; + uint32_t sum = 0; + + // There's a weird bug in some versions of GCC where building with -O2 or + // higher will cause the initialization here to get optimized away, and + // lead to the compiler warning that this variable is used uninitialized. + // Using 'volatile' here short-circuits that optimization and fixes the + // warning. + volatile uint32_t addl_pseudo = 0; + + if ( len % 2 == 1 ) + // Add in pad byte. + sum = htons(((const u_char*) tp)[tcp_len - 1] << 8); + else + sum = 0; + + sum = ones_complement_checksum((void*) &ip->ip_src.s_addr, 4, sum); + sum = ones_complement_checksum((void*) &ip->ip_dst.s_addr, 4, sum); + + addl_pseudo = (htons(IPPROTO_TCP) << 16) | htons((unsigned short) tcp_len); + + sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum); + sum = ones_complement_checksum((void*) tp, tcp_len, sum); + + return sum; +} + +void send_pkt(int s, struct in_addr from, int from_port, uint32_t from_seq, + struct in_addr to, int to_port, uint32_t to_seq, + int size, int redundancy, int delay, int flags, + const char *inject) +{ + int cc; + int pktlen = 40 + size; + const int max_injection_size = 4096; + char *pkt = malloc(pktlen + max_injection_size + 1024 /* slop */); + struct ip *ip = (struct ip *) pkt; + struct tcphdr *tcp = (struct tcphdr *) &pkt[20]; + + if ( ! pkt ) + pgripe("couldn't malloc memory"); + + if ( inject && *inject ) { + size = strlen(inject); + + if ( size > max_injection_size ) + gripe("injection text too large%s", ""); + + pktlen = 40 + size; + } + + memset(pkt, 0, pktlen); + + ip->ip_v = IPVERSION; + ip->ip_len = pktlen; /* on FreeBSD, don't use htons(); YMMV */ + ip->ip_off = 0; + ip->ip_src = from; + ip->ip_dst = to; + ip->ip_hl = 5; + ip->ip_p = IPPROTO_TCP; + ip->ip_ttl = 255; + ip->ip_id = 0; + + ip->ip_sum = in_cksum((uint16_t *) ip, sizeof(*ip)); + + if (ip->ip_sum == 0) + ip->ip_sum = 0xffff; + + tcp->th_sport = htons(from_port); + tcp->th_dport = htons(to_port); + tcp->th_seq = htonl(from_seq); + tcp->th_ack = htonl(to_seq); + tcp->th_off = 5; + tcp->th_flags = flags; + tcp->th_win = 0; + tcp->th_urp = 0; + tcp->th_sum = 0; + + if ( inject && *inject ) { + char *payload = &pkt[40]; + strcpy(payload, inject); + + } else if ( size > 0 ) + { + const char *fill_string = + (inject && *inject) ? inject : "BRO-RST\n"; + char *payload = &pkt[40]; + int n = strlen(fill_string); + int i; + for ( i = size; i > n + 1; i -= n ) + { + strcpy(payload, fill_string); + payload += n; + } + + for ( ; i > 0; --i ) + *(payload++) = '\n'; + } + + tcp->th_sum = ~tcp_checksum(ip, tcp, size); + + while ( redundancy-- > 0 ) + { + cc = send(s, (char *) ip, pktlen, 0); + if (cc < 0 || cc != pktlen) + pgripe("problem in sendto()"); + usleep(delay * 1000); + } + + free(pkt); +} + +void terminate(int s, const char *from_addr, int from_port, uint32_t from_seq, + const char *to_addr, int to_port, uint32_t to_seq, + int num, int redundancy, int stride, int delay, + const char *inject) +{ + struct sockaddr_in where_from, where_to; + struct sockaddr_in *from = (struct sockaddr_in *) &where_from; + struct sockaddr_in *to = (struct sockaddr_in *) &where_to; + + memset(from, 0, sizeof(*from)); + memset(to, 0, sizeof(*to)); +#ifdef SIN_LEN + from->sin_len = to->sin_len = sizeof(*to); +#endif /* SIN_LEN */ + from->sin_family = to->sin_family = AF_INET; + + if ( inet_aton(from_addr, (struct in_addr *) &from->sin_addr) == 0 ) + gripe("bad from address %s", from_addr); + if ( inet_aton(to_addr, (struct in_addr *) &to->sin_addr) == 0 ) + gripe("bad to address %s", to_addr); + + if ( connect(s, (struct sockaddr *) &where_to, sizeof(where_to)) < 0 ) + pgripe("can't connect"); + + while ( num-- > 0 ) + { + send_pkt(s, from->sin_addr, from_port, from_seq, + to->sin_addr, to_port, to_seq, 0, redundancy, delay, + (*inject ? 0 : TH_RST) | TH_ACK, inject); + + if ( num > 0 && stride > 1 ) + send_pkt(s, from->sin_addr, from_port, from_seq, + to->sin_addr, to_port, to_seq, stride, + redundancy, delay, TH_ACK, inject); + + from_seq += stride; + } +} + +void usage() +{ +#if defined(__linux__) + fprintf(stderr, "%s [-R] [-I text-to-inject] [-i interface] [-d delay-msec] [-n num] [-r redundancy] [-s stride] from_addr from_port from_seq to_addr to_port to_seq\n", prog_name); +#else + fprintf(stderr, "%s [-R] [-I text-to-inject] [-d delay-msec] [-n num] [-r redundancy] [-s stride] from_addr from_port from_seq to_addr to_port to_seq\n", prog_name); +#endif + exit(0); +} + +int main(int argc, char **argv) +{ + extern char* optarg; + extern int optind, opterr; + const char *from_addr, *to_addr; + char inject[8192]; + int from_port, to_port; + uint32_t from_seq, to_seq; + int delay = 0.0; + int redundancy = 1; + int num = 1; + int stride = 1; + int reverse = 0; + int s; + int on = 1; + int op; + + prog_name = argv[0]; + + opterr = 0; + + inject[0] = 0; + +#if defined(__linux__) + char *interface = NULL; + + while ( (op = getopt(argc, argv, "RI:i:d:n:r:s:")) != EOF ) +#else + while ( (op = getopt(argc, argv, "RI:d:n:r:s:")) != EOF ) +#endif + switch ( op ) { + case 'R': + reverse = 1; + break; + + case 'I': + { + char *ap = optarg; + char *ip; + for ( ip = inject; *ap; ++ip, ++ap ) { + if ( ap[0] == '\\' && ap[1] == 'n' ) + *ip = '\n', ++ap; + else + *ip = *ap; + } + } + break; + +#if defined(__linux__) + case 'i': + interface = optarg; + break; +#endif + + case 'd': + delay = atoi(optarg); + break; + + case 'n': + num = atoi(optarg); + break; + + case 'r': + redundancy = atoi(optarg); + break; + + case 's': + stride = atoi(optarg); + break; + + default: + usage(); + break; + } + + if ( argc - optind != 6 ) + usage(); + + s = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); + if ( s < 0 ) + pgripe("couldn't create raw socket"); + + if ( setuid(getuid()) ) + pgripe("couldn't lower privileges"); + + if ( setsockopt(s, 0, IP_HDRINCL, (char *) &on, sizeof(on)) < 0 ) + pgripe("can't turn on IP_HDRINCL"); + +#if defined(__linux__) + if ( interface ){ + if ( setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, interface, strlen(interface)) < 0 ) + pgripe("can't set interface"); + } +#endif + + from_addr = argv[optind++]; + from_port = atoi(argv[optind++]); + from_seq = strtoul(argv[optind++], 0, 10); + + to_addr = argv[optind++]; + to_port = atoi(argv[optind++]); + to_seq = strtoul(argv[optind++], 0, 10); + + if ( reverse ) + terminate(s, to_addr, to_port, to_seq, + from_addr, from_port, from_seq, + num, redundancy, stride, delay, inject); + else + terminate(s, from_addr, from_port, from_seq, + to_addr, to_port, to_seq, + num, redundancy, stride, delay, inject); + + return 0; +} diff --git a/auxil/zeek-aux/testing/.gitignore b/auxil/zeek-aux/testing/.gitignore new file mode 100644 index 0000000000..91ad861d09 --- /dev/null +++ b/auxil/zeek-aux/testing/.gitignore @@ -0,0 +1,3 @@ +.btest.failed.dat +diag.log +.tmp diff --git a/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-c-header/header.h b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-c-header/header.h new file mode 100644 index 0000000000..7a160c5abb --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-c-header/header.h @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#define ZEEK_VERSION "1.0.0-2" /* with comment */ +#define ZEEK_VERSION "1.0.0-2" /* with comment */ + #define FOO_VERSION "1.0.0-2" // another comment +... diff --git a/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-python-package/__init__.py b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-python-package/__init__.py new file mode 100644 index 0000000000..874e07ea4a --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-python-package/__init__.py @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +__version__ = "1.0.0.dev2", # with comment +__version__ = "1.0.0.dev2", # another comment +__version__ = "1.0.0.dev2", # Python style +__version__ = "0.0.1.nope" # should not change +version = "0.0.1" # should not change +print('Additional change') diff --git a/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-rst/test.rst b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-rst/test.rst new file mode 100644 index 0000000000..2464164fcb --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-rst/test.rst @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +.. |version| replace:: 1.0.0-2 +.. |version| replace:: 1.0.0-2 +.. |version| replace:: 1.0.0-2 +.. |version| replace:: 1.0.0-2 +.. |version| replace:: 1.0.0-2 +... diff --git a/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-script/test.pl b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-script/test.pl new file mode 100644 index 0000000000..35961a9c53 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-script/test.pl @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +VERSION="1.0.0-2" +VERSION="1.0.0-2" +VERSION="1.0.0-2" +VERSION="1.0.0-2" + VERSION = "1.0.0-2" # with some comment + VERSION = "2.0.0-nope" # with some comment +... diff --git a/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-setup-py/setup.py b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-setup-py/setup.py new file mode 100644 index 0000000000..ac27caeb8f --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/update-changes.replace-version-in-setup-py/setup.py @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +version = "1.0.0.dev2", # short +version = "1.0.0.dev2", # with dev update +version = "1.0.0.dev2", # long +version = "1.0.0.dev2", # long with dev update +version = "1.0.0.dev2", # Python style +version = "0.0.1.nope" # should not change, invalid suffix +print('Additional change') diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-fields-header b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-fields-header new file mode 100644 index 0000000000..9a64eb71a4 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-fields-header @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +zeek-cut: bad log header (missing #fields line) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-sep-header b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-sep-header new file mode 100644 index 0000000000..884d35537e --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-sep-header @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CjhGID4nQcgTWjvg4c tcp +CCvvfg3TEfuqmmG4bh tcp +CsRx2w45OKnoww6xl4 tcp +CRJuHdVW0XPVINV8a tcp +CXWv6p3arKYeMETxOg tcp diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-separator new file mode 100644 index 0000000000..0de1251424 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/missing-separator @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +zeek-cut: bad log header (invalid #separator line) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-column b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-column new file mode 100644 index 0000000000..0865b11624 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-column @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-not-column b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-not-column new file mode 100644 index 0000000000..0865b11624 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-not-column @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-show b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-show new file mode 100644 index 0000000000..0865b11624 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/no-header-show @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/null-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/null-separator new file mode 100644 index 0000000000..0de1251424 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.bad-logs/null-separator @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +zeek-cut: bad log header (invalid #separator line) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/all b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/all new file mode 100644 index 0000000000..5bbd83a4af --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/all @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329327783.316897 CjhGID4nQcgTWjvg4c 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 CCvvfg3TEfuqmmG4bh 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 CsRx2w45OKnoww6xl4 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 CRJuHdVW0XPVINV8a 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 CXWv6p3arKYeMETxOg 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 CPbrpk1qSsw6ESzHV4 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/different-col-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/different-col-order new file mode 100644 index 0000000000..b86c6ba0a1 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/different-col-order @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329843175.736107 CjhGID4nQcgTWjvg4c +1329843179.871641 CCvvfg3TEfuqmmG4bh +1329843194.151526 CsRx2w45OKnoww6xl4 +1329843197.783443 CRJuHdVW0XPVINV8a +1329843161.968492 CXWv6p3arKYeMETxOg +1329843175.736107 CjhGID4nQcgTWjvg4c +1329843179.871641 CCvvfg3TEfuqmmG4bh +1329843194.151526 CsRx2w45OKnoww6xl4 +1329843197.783443 CRJuHdVW0XPVINV8a +1329843161.968492 CXWv6p3arKYeMETxOg diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/nondefault-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/nondefault-separator new file mode 100644 index 0000000000..60db9ad1b4 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/nondefault-separator @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +tcp CjhGID4nQcgTWjvg4c +tcp CCvvfg3TEfuqmmG4bh +tcp CsRx2w45OKnoww6xl4 +tcp CRJuHdVW0XPVINV8a +tcp CXWv6p3arKYeMETxOg +tcp,CNbXUV0IZ29or3MK6 +tcp,CJ8woc3c6CfBLdiyp5 +tcp,CXlgj54ftP8Yc2GSnb +tcp,Czw8Gd1zEVn3Xz5x7i +tcp,Cys4aQ15qDqHzsIk3l diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one new file mode 100644 index 0000000000..a33e087382 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CjhGID4nQcgTWjvg4c +CCvvfg3TEfuqmmG4bh +CsRx2w45OKnoww6xl4 +CRJuHdVW0XPVINV8a +CXWv6p3arKYeMETxOg diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one-nonexistent-1 b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one-nonexistent-1 new file mode 100644 index 0000000000..c1d55b6537 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one-nonexistent-1 @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CjhGID4nQcgTWjvg4c tcp +CCvvfg3TEfuqmmG4bh tcp +CsRx2w45OKnoww6xl4 tcp +CRJuHdVW0XPVINV8a tcp +CXWv6p3arKYeMETxOg tcp +CjhGID4nQcgTWjvg4c +CCvvfg3TEfuqmmG4bh +CsRx2w45OKnoww6xl4 +CRJuHdVW0XPVINV8a +CXWv6p3arKYeMETxOg +CPbrpk1qSsw6ESzHV4 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one-nonexistent-2 b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one-nonexistent-2 new file mode 100644 index 0000000000..a86c6c34df --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/one-nonexistent-2 @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CjhGID4nQcgTWjvg4c +CCvvfg3TEfuqmmG4bh +CsRx2w45OKnoww6xl4 +CRJuHdVW0XPVINV8a +CXWv6p3arKYeMETxOg +CPbrpk1qSsw6ESzHV4 +CjhGID4nQcgTWjvg4c tcp +CCvvfg3TEfuqmmG4bh tcp +CsRx2w45OKnoww6xl4 tcp +CRJuHdVW0XPVINV8a tcp +CXWv6p3arKYeMETxOg tcp diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/only b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/only new file mode 100644 index 0000000000..863442af67 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/only @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +79.26.245.236 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/swap-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/swap-order new file mode 100644 index 0000000000..082542658a --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.columns/swap-order @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +tcp CjhGID4nQcgTWjvg4c +tcp CCvvfg3TEfuqmmG4bh +tcp CsRx2w45OKnoww6xl4 +tcp CRJuHdVW0XPVINV8a +tcp CXWv6p3arKYeMETxOg diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.help/show-help b/auxil/zeek-aux/testing/Baseline/zeek-cut.help/show-help new file mode 100644 index 0000000000..90f5f902c6 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.help/show-help @@ -0,0 +1,25 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + +zeek-cut [options] [] + +Extracts the given columns from ASCII Zeek logs on standard input, and outputs +them to standard output. If no columns are given, all are selected. +By default, zeek-cut does not include format header blocks in the output. + +Example: cat conn.log | zeek-cut -d ts id.orig_h id.orig_p + + -c Include the first format header block in the output. + -C Include all format header blocks in the output. + -m Include the first format header blocks in the output in minimal view. + -M Include all format header blocks in the output in minimal view. + -d Convert time values into human-readable format. + -D Like -d, but specify format for time (see strftime(3) for syntax). + -F Sets a different output field separator character. + -h Show help. + -n Print all fields *except* those specified. + -u Like -d, but print timestamps in UTC instead of local time. + -U Like -D, but print timestamps in UTC instead of local time. + +For time conversion option -d or -u, the format string can be specified by +setting an environment variable ZEEK_CUT_TIMEFMT. + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-c-m-opts-m b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-c-m-opts-m new file mode 100644 index 0000000000..ea23ee7204 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-c-m-opts-m @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid ts +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-c-opts-c b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-c-opts-c new file mode 100644 index 0000000000..2a23436ed9 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-c-opts-c @@ -0,0 +1,20 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-opts-all b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-opts-all new file mode 100644 index 0000000000..f2f743000b --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-opts-all @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid ts +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +uid ts +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-opts-one b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-opts-one new file mode 100644 index 0000000000..ea23ee7204 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/both-m-opts-one @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid ts +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/different-col-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/different-col-order new file mode 100644 index 0000000000..f2d5f85784 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/different-col-order @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents ts +CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) 1329843175.736107 +CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) 1329843179.871641 +CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) 1329843194.151526 +CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) 1329843197.783443 +CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) 1329843161.968492 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/different-col-order-some b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/different-col-order-some new file mode 100644 index 0000000000..5cc0e9eae6 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/different-col-order-some @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid +1329843175.736107 CjhGID4nQcgTWjvg4c +1329843179.871641 CCvvfg3TEfuqmmG4bh +1329843194.151526 CsRx2w45OKnoww6xl4 +1329843197.783443 CRJuHdVW0XPVINV8a +1329843161.968492 CXWv6p3arKYeMETxOg +ts uid +1329843175.736107 CjhGID4nQcgTWjvg4c +1329843179.871641 CCvvfg3TEfuqmmG4bh +1329843194.151526 CsRx2w45OKnoww6xl4 +1329843197.783443 CRJuHdVW0XPVINV8a +1329843161.968492 CXWv6p3arKYeMETxOg diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/nondefault-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/nondefault-separator new file mode 100644 index 0000000000..4fba48abf2 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/nondefault-separator @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +proto uid +tcp CjhGID4nQcgTWjvg4c +tcp CCvvfg3TEfuqmmG4bh +tcp CsRx2w45OKnoww6xl4 +tcp CRJuHdVW0XPVINV8a +tcp CXWv6p3arKYeMETxOg +proto,uid +tcp,CNbXUV0IZ29or3MK6 +tcp,CJ8woc3c6CfBLdiyp5 +tcp,CXlgj54ftP8Yc2GSnb +tcp,Czw8Gd1zEVn3Xz5x7i +tcp,Cys4aQ15qDqHzsIk3l diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/one-all-nocols b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/one-all-nocols new file mode 100644 index 0000000000..475bd22c49 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/one-all-nocols @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-1st-withcols b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-1st-withcols new file mode 100644 index 0000000000..ea23ee7204 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-1st-withcols @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid ts +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-1st-withcols-custom-sep b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-1st-withcols-custom-sep new file mode 100644 index 0000000000..fe93adea12 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-1st-withcols-custom-sep @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid,ts +CjhGID4nQcgTWjvg4c,1329843175.736107 +CCvvfg3TEfuqmmG4bh,1329843179.871641 +CsRx2w45OKnoww6xl4,1329843194.151526 +CRJuHdVW0XPVINV8a,1329843197.783443 +CXWv6p3arKYeMETxOg,1329843161.968492 +CjhGID4nQcgTWjvg4c,1329327783.316897 +CCvvfg3TEfuqmmG4bh,1329327786.524332 +CsRx2w45OKnoww6xl4,1329327787.289095 +CRJuHdVW0XPVINV8a,1329327795.571921 +CXWv6p3arKYeMETxOg,1329327777.822004 +CPbrpk1qSsw6ESzHV4,1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-all-withcols b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-all-withcols new file mode 100644 index 0000000000..f2f743000b --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-all-withcols @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid ts +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +uid ts +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-all-withcols-custom-sep b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-all-withcols-custom-sep new file mode 100644 index 0000000000..6bd8a322a0 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.minimal-view/two-all-withcols-custom-sep @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +uid,ts +CjhGID4nQcgTWjvg4c,1329843175.736107 +CCvvfg3TEfuqmmG4bh,1329843179.871641 +CsRx2w45OKnoww6xl4,1329843194.151526 +CRJuHdVW0XPVINV8a,1329843197.783443 +CXWv6p3arKYeMETxOg,1329843161.968492 +uid,ts +CjhGID4nQcgTWjvg4c,1329327783.316897 +CCvvfg3TEfuqmmG4bh,1329327786.524332 +CsRx2w45OKnoww6xl4,1329327787.289095 +CRJuHdVW0XPVINV8a,1329327795.571921 +CXWv6p3arKYeMETxOg,1329327777.822004 +CPbrpk1qSsw6ESzHV4,1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.no-options/no-opts b/auxil/zeek-aux/testing/Baseline/zeek-cut.no-options/no-opts new file mode 100644 index 0000000000..0b307f0c62 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.no-options/no-opts @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.no-options/tsv b/auxil/zeek-aux/testing/Baseline/zeek-cut.no-options/tsv new file mode 100644 index 0000000000..ec015b2fe5 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.no-options/tsv @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/different-col-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/different-col-order new file mode 100644 index 0000000000..646f97cf78 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/different-col-order @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-all b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-all new file mode 100644 index 0000000000..ec015b2fe5 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-all @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-none b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-none new file mode 100644 index 0000000000..5bbd83a4af --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-none @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329327783.316897 CjhGID4nQcgTWjvg4c 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 CCvvfg3TEfuqmmG4bh 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 CsRx2w45OKnoww6xl4 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 CRJuHdVW0XPVINV8a 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 CXWv6p3arKYeMETxOg 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 CPbrpk1qSsw6ESzHV4 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one new file mode 100644 index 0000000000..b489027110 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329327783.316897 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-nonexistent b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-nonexistent new file mode 100644 index 0000000000..b489027110 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-nonexistent @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329327783.316897 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-show-header b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-show-header new file mode 100644 index 0000000000..b80ced0c19 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-show-header @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-15-51 +#fields ts id.orig_h id.orig_p id.resp_h +#types time addr port addr +1329327783.316897 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-twice b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-twice new file mode 100644 index 0000000000..b489027110 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-one-twice @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329327783.316897 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-only b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-only new file mode 100644 index 0000000000..a21c51bcbc --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-only @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-two-swapped b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-two-swapped new file mode 100644 index 0000000000..cec61a9f85 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.not-columns/not-two-swapped @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1329327783.316897 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 +1329327786.524332 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 +1329327787.289095 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 +1329327795.571921 2001:470:4867:99::21 55785 +1329327777.822004 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 +1329327800.017649 2001:470:4867:99::21 55647 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/nondefault-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/nondefault-separator new file mode 100644 index 0000000000..d64376e324 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/nondefault-separator @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +tcp:CjhGID4nQcgTWjvg4c +tcp:CCvvfg3TEfuqmmG4bh +tcp:CsRx2w45OKnoww6xl4 +tcp:CRJuHdVW0XPVINV8a +tcp:CXWv6p3arKYeMETxOg +tcp:CNbXUV0IZ29or3MK6 +tcp:CJ8woc3c6CfBLdiyp5 +tcp:CXlgj54ftP8Yc2GSnb +tcp:Czw8Gd1zEVn3Xz5x7i +tcp:Cys4aQ15qDqHzsIk3l diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/only-column b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/only-column new file mode 100644 index 0000000000..863442af67 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/only-column @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +79.26.245.236 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/show-header b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/show-header new file mode 100644 index 0000000000..83a4ee23d7 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/show-header @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields.ts.uid.id.orig_h.id.orig_p.id.resp_h.id.resp_p.proto.service.duration.orig_bytes.resp_bytes.conn_state.local_orig.missed_bytes.history.orig_pkts.orig_ip_bytes.resp_pkts.resp_ip_bytes.tunnel_parents +#types.time.string.addr.port.addr.port.enum.string.interval.count.count.string.bool.count.string.count.count.count.count.set[string] +1329843175.736107.CjhGID4nQcgTWjvg4c.141.142.220.235.37604.199.233.217.249.56666.tcp.ftp-data.0.112432.0.342.SF.-.0.ShAdfFa.4.216.4.562.(empty) +1329843179.871641.CCvvfg3TEfuqmmG4bh.141.142.220.235.59378.199.233.217.249.56667.tcp.ftp-data.0.111218.0.77.SF.-.0.ShAdfFa.4.216.4.297.(empty) +1329843194.151526.CsRx2w45OKnoww6xl4.199.233.217.249.61920.141.142.220.235.33582.tcp.ftp-data.0.056211.342.0.SF.-.0.ShADaFf.5.614.3.164.(empty) +1329843197.783443.CRJuHdVW0XPVINV8a.199.233.217.249.61918.141.142.220.235.37835.tcp.ftp-data.0.056005.77.0.SF.-.0.ShADaFf.5.349.3.164.(empty) +1329843161.968492.CXWv6p3arKYeMETxOg.141.142.220.235.50003.199.233.217.249.21.tcp.ftp.38.055625.180.3146.SF.-.0.ShAdDfFa.38.2164.25.4458.(empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/some-columns b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/some-columns new file mode 100644 index 0000000000..b813fcf756 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.ofs/some-columns @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CjhGID4nQcgTWjvg4c,1329843175.736107 +CCvvfg3TEfuqmmG4bh,1329843179.871641 +CsRx2w45OKnoww6xl4,1329843194.151526 +CRJuHdVW0XPVINV8a,1329843197.783443 +CXWv6p3arKYeMETxOg,1329843161.968492 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/both-c-opts-all b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/both-c-opts-all new file mode 100644 index 0000000000..4741d52d4d --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/both-c-opts-all @@ -0,0 +1,30 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +#close 2014-04-01-23-15-49 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-15-51 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 +#close 2014-04-01-23-15-51 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/both-c-opts-one b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/both-c-opts-one new file mode 100644 index 0000000000..2a23436ed9 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/both-c-opts-one @@ -0,0 +1,20 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/different-col-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/different-col-order new file mode 100644 index 0000000000..0a74f4b44d --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/different-col-order @@ -0,0 +1,29 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +#close 2014-04-01-23-15-49 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conntimelast +#open 2014-04-01-23-15-49 +#fields uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents ts +#types string addr port addr port enum string interval count count string bool count string count count count count set[string] time +CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) 1329843175.736107 +CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) 1329843179.871641 +CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) 1329843194.151526 +CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) 1329843197.783443 +CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) 1329843161.968492 +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/different-col-order-some b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/different-col-order-some new file mode 100644 index 0000000000..c0837e4553 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/different-col-order-some @@ -0,0 +1,29 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts uid +#types time string +1329843175.736107 CjhGID4nQcgTWjvg4c +1329843179.871641 CCvvfg3TEfuqmmG4bh +1329843194.151526 CsRx2w45OKnoww6xl4 +1329843197.783443 CRJuHdVW0XPVINV8a +1329843161.968492 CXWv6p3arKYeMETxOg +#close 2014-04-01-23-15-49 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conntimelast +#open 2014-04-01-23-15-49 +#fields ts uid +#types time string +1329843175.736107 CjhGID4nQcgTWjvg4c +1329843179.871641 CCvvfg3TEfuqmmG4bh +1329843194.151526 CsRx2w45OKnoww6xl4 +1329843197.783443 CRJuHdVW0XPVINV8a +1329843161.968492 CXWv6p3arKYeMETxOg +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/nondefault-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/nondefault-separator new file mode 100644 index 0000000000..37aa77c6ff --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/nondefault-separator @@ -0,0 +1,29 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields proto uid +#types enum string +tcp CjhGID4nQcgTWjvg4c +tcp CCvvfg3TEfuqmmG4bh +tcp CsRx2w45OKnoww6xl4 +tcp CRJuHdVW0XPVINV8a +tcp CXWv6p3arKYeMETxOg +#close 2014-04-01-23-15-49 +#separator , +#set_separator,, +#empty_field,(empty) +#unset_field,- +#path,conn +#open,2014-06-30-16-10-54 +#fields,proto,uid +#types,enum,string +tcp,CNbXUV0IZ29or3MK6 +tcp,CJ8woc3c6CfBLdiyp5 +tcp,CXlgj54ftP8Yc2GSnb +tcp,Czw8Gd1zEVn3Xz5x7i +tcp,Cys4aQ15qDqHzsIk3l +#close,2014-06-30-16-10-55 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/one-1st-nocols b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/one-1st-nocols new file mode 100644 index 0000000000..9c08053921 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/one-1st-nocols @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/one-all-nocols b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/one-all-nocols new file mode 100644 index 0000000000..e38d3d35c5 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/one-all-nocols @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/two-1st-withcols b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/two-1st-withcols new file mode 100644 index 0000000000..2a23436ed9 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/two-1st-withcols @@ -0,0 +1,20 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/two-all-withcols b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/two-all-withcols new file mode 100644 index 0000000000..4741d52d4d --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.show-header/two-all-withcols @@ -0,0 +1,30 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329843175.736107 +CCvvfg3TEfuqmmG4bh 1329843179.871641 +CsRx2w45OKnoww6xl4 1329843194.151526 +CRJuHdVW0XPVINV8a 1329843197.783443 +CXWv6p3arKYeMETxOg 1329843161.968492 +#close 2014-04-01-23-15-49 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-15-51 +#fields uid ts +#types string time +CjhGID4nQcgTWjvg4c 1329327783.316897 +CCvvfg3TEfuqmmG4bh 1329327786.524332 +CsRx2w45OKnoww6xl4 1329327787.289095 +CRJuHdVW0XPVINV8a 1329327795.571921 +CXWv6p3arKYeMETxOg 1329327777.822004 +CPbrpk1qSsw6ESzHV4 1329327800.017649 +#close 2014-04-01-23-15-51 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/invalid-time-field b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/invalid-time-field new file mode 100644 index 0000000000..5a52a25d85 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/invalid-time-field @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +hello a +-12345.123456 b +77777777777777777777 c + d +123456789 e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/invalid-time-field.err b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/invalid-time-field.err new file mode 100644 index 0000000000..e0f5b2956e --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/invalid-time-field.err @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +zeek-cut: time field is not valid: hello +zeek-cut: time value out-of-range: -12345.123456 +zeek-cut: time value out-of-range: 77777777777777777777 +zeek-cut: time field is not valid: +zeek-cut: time field is not valid: 123456789 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/missing-types-header b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/missing-types-header new file mode 100644 index 0000000000..2fbc8ddfe6 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/missing-types-header @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +zeek-cut: bad log header (missing #types line) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/no-header-time b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/no-header-time new file mode 100644 index 0000000000..0865b11624 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-bad-logs/no-header-time @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/both-1 b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/both-1 new file mode 100644 index 0000000000..bd0d157f27 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/both-1 @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2012-02-21T16:52:55+0000 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +2012-02-21T16:52:59+0000 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +2012-02-21T16:53:14+0000 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +2012-02-21T16:53:17+0000 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +2012-02-21T16:52:41+0000 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/both-2 b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/both-2 new file mode 100644 index 0000000000..4f92e4068f --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/both-2 @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2012-02-21T08:52:55-0800 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +2012-02-21T08:52:59-0800 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +2012-02-21T08:53:14-0800 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +2012-02-21T08:53:17-0800 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +2012-02-21T08:52:41-0800 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/different-col-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/different-col-order new file mode 100644 index 0000000000..4c57810e32 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/different-col-order @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2012-02-21T08:52:55-0800 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +2012-02-21T08:52:59-0800 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +2012-02-21T08:53:14-0800 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +2012-02-21T08:53:17-0800 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +2012-02-21T08:52:41-0800 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) 2012-02-21T08:52:55-0800 +CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) 2012-02-21T08:52:59-0800 +CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) 2012-02-21T08:53:14-0800 +CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) 2012-02-21T08:53:17-0800 +CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) 2012-02-21T08:52:41-0800 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/local b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/local new file mode 100644 index 0000000000..4f92e4068f --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/local @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2012-02-21T08:52:55-0800 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +2012-02-21T08:52:59-0800 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +2012-02-21T08:53:14-0800 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +2012-02-21T08:53:17-0800 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +2012-02-21T08:52:41-0800 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/missing-header b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/missing-header new file mode 100644 index 0000000000..0865b11624 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/missing-header @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. + + + + + diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/multiple-times b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/multiple-times new file mode 100644 index 0000000000..3fe073a34a --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/multiple-times @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2012-02-21T08:52:55-0800 tcp 2012-02-21T08:53:05-0800 +2012-02-21T08:52:59-0800 udp 2012-02-21T08:52:59-0800 +2012-02-21T08:53:14-0800 tcp 2012-02-21T08:53:24-0800 +2012-02-21T08:53:24-0800 tcp 2012-02-21T08:54:15-0800 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/no-timestamps b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/no-timestamps new file mode 100644 index 0000000000..863442af67 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/no-timestamps @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +79.26.245.236 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/time-last-col b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/time-last-col new file mode 100644 index 0000000000..9a73503143 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/time-last-col @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) 2012-02-21T08:52:55-0800 +CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) 2012-02-21T08:52:59-0800 +CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) 2012-02-21T08:53:14-0800 +CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) 2012-02-21T08:53:17-0800 +CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) 2012-02-21T08:52:41-0800 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/utc b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/utc new file mode 100644 index 0000000000..bd0d157f27 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/utc @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2012-02-21T16:52:55+0000 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +2012-02-21T16:52:59+0000 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +2012-02-21T16:53:14+0000 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +2012-02-21T16:53:17+0000 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +2012-02-21T16:52:41+0000 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/zero b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/zero new file mode 100644 index 0000000000..186b7a87cd --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-conversion/zero @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1970-01-01T00:00:00+0000 CjhGID4nQcgTWjvg4c 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +2012-02-15T17:43:06+0000 CCvvfg3TEfuqmmG4bh 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +2012-02-15T17:43:07+0000 CsRx2w45OKnoww6xl4 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +2012-02-15T17:43:15+0000 CRJuHdVW0XPVINV8a 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +2012-02-15T17:42:57+0000 CXWv6p3arKYeMETxOg 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +2012-02-15T17:43:20+0000 CPbrpk1qSsw6ESzHV4 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-local b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-local new file mode 100644 index 0000000000..bc370043e5 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-local @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0852_120221 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +0852_120221 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +0853_120221 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +0853_120221 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +0852_120221 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-local-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-local-fmt new file mode 100644 index 0000000000..67218e781e --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-local-fmt @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +08 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +08 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +08 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +08 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +08 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-utc b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-utc new file mode 100644 index 0000000000..2fb696a5fe --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-utc @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1652_120221 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1652_120221 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1653_120221 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1653_120221 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1652_120221 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-utc-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-utc-fmt new file mode 100644 index 0000000000..3d8d27e449 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt-env/env-utc-fmt @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +16 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +16 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +16 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +16 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +16 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-fmt new file mode 100644 index 0000000000..67218e781e --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-fmt @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +08 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +08 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +08 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +08 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +08 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-fmt-utc b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-fmt-utc new file mode 100644 index 0000000000..3d8d27e449 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-fmt-utc @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +16 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +16 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +16 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +16 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +16 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-utc-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-utc-fmt new file mode 100644 index 0000000000..3d8d27e449 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/local-utc-fmt @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +16 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +16 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +16 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +16 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +16 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-fmt new file mode 100644 index 0000000000..3d8d27e449 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-fmt @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +16 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +16 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +16 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +16 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +16 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-fmt-local b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-fmt-local new file mode 100644 index 0000000000..67218e781e --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-fmt-local @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +08 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +08 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +08 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +08 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +08 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-local-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-local-fmt new file mode 100644 index 0000000000..67218e781e --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-fmt/utc-local-fmt @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +08 52 12 02 21 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +08 52 12 02 21 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +08 53 12 02 21 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +08 53 12 02 21 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +08 52 12 02 21 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/different-col-order b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/different-col-order new file mode 100644 index 0000000000..4a39a899a5 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/different-col-order @@ -0,0 +1,29 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields proto ts id.orig_h +#types enum string addr +tcp 2012-02-21T08:52:55-0800 141.142.220.235 +tcp 2012-02-21T08:52:59-0800 141.142.220.235 +tcp 2012-02-21T08:53:14-0800 199.233.217.249 +tcp 2012-02-21T08:53:17-0800 199.233.217.249 +tcp 2012-02-21T08:52:41-0800 141.142.220.235 +#close 2014-04-01-23-15-49 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conntimelast +#open 2014-04-01-23-15-49 +#fields proto ts id.orig_h +#types enum string addr +tcp 2012-02-21T08:52:55-0800 141.142.220.235 +tcp 2012-02-21T08:52:59-0800 141.142.220.235 +tcp 2012-02-21T08:53:14-0800 199.233.217.249 +tcp 2012-02-21T08:53:17-0800 199.233.217.249 +tcp 2012-02-21T08:52:41-0800 141.142.220.235 +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/different-field-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/different-field-separator new file mode 100644 index 0000000000..743f3cdffa --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/different-field-separator @@ -0,0 +1,29 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields proto ts id.orig_h +#types enum string addr +tcp 2012-02-21T08:52:55-0800 141.142.220.235 +tcp 2012-02-21T08:52:59-0800 141.142.220.235 +tcp 2012-02-21T08:53:14-0800 199.233.217.249 +tcp 2012-02-21T08:53:17-0800 199.233.217.249 +tcp 2012-02-21T08:52:41-0800 141.142.220.235 +#close 2014-04-01-23-15-49 +#separator , +#set_separator,, +#empty_field,(empty) +#unset_field,- +#path,conn +#open,2014-06-30-16-10-54 +#fields,proto,ts,id.orig_h +#types,enum,string,addr +tcp,2012-02-21T08:52:55-0800,141.142.220.235 +tcp,2012-02-21T08:52:59-0800,141.142.220.235 +tcp,2012-02-21T08:53:14-0800,199.233.217.249 +tcp,2012-02-21T08:52:41-0800,141.142.220.235 +tcp,2012-02-21T08:53:17-0800,199.233.217.249 +#close,2014-06-30-16-10-55 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/missing-ts-one-file b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/missing-ts-one-file new file mode 100644 index 0000000000..dc72c7c5dc --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/missing-ts-one-file @@ -0,0 +1,25 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts id.orig_h +#types string addr +2012-02-21T08:52:55-0800 141.142.220.235 +2012-02-21T08:52:59-0800 141.142.220.235 +2012-02-21T08:53:14-0800 199.233.217.249 +2012-02-21T08:53:17-0800 199.233.217.249 +2012-02-21T08:52:41-0800 141.142.220.235 +#close 2014-04-01-23-15-49 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path onecolumn +#open 2014-04-01-23-15-59 +#fields id.orig_h +#types addr + 79.26.245.236 +#close 2014-04-01-23-15-59 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/multiple-times b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/multiple-times new file mode 100644 index 0000000000..e01fef6158 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/multiple-times @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-16-29 +#fields proto ts2 ts1 +#types enum string string +tcp 2012-02-21T08:53:05-0800 2012-02-21T08:52:55-0800 +udp 2012-02-21T08:52:59-0800 2012-02-21T08:52:59-0800 +tcp 2012-02-21T08:53:24-0800 2012-02-21T08:53:14-0800 +tcp 2012-02-21T08:54:15-0800 2012-02-21T08:53:24-0800 +#close 2014-04-01-23-16-29 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/nondefault-separator b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/nondefault-separator new file mode 100644 index 0000000000..580e4bf18f --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/nondefault-separator @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator , +#set_separator,, +#empty_field,(empty) +#unset_field,- +#path,conn +#open,2014-06-30-16-10-54 +#fields,proto,ts,id.orig_h +#types,enum,string,addr +tcp,2012-02-21T08:52:55-0800,141.142.220.235 +tcp,2012-02-21T08:52:59-0800,141.142.220.235 +tcp,2012-02-21T08:53:14-0800,199.233.217.249 +tcp,2012-02-21T08:52:41-0800,141.142.220.235 +tcp,2012-02-21T08:53:17-0800,199.233.217.249 +#close,2014-06-30-16-10-55 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/ts-twice b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/ts-twice new file mode 100644 index 0000000000..e12215f6d7 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/ts-twice @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts id.orig_h ts +#types string addr string +2012-02-21T08:52:55-0800 141.142.220.235 2012-02-21T08:52:55-0800 +2012-02-21T08:52:59-0800 141.142.220.235 2012-02-21T08:52:59-0800 +2012-02-21T08:53:14-0800 199.233.217.249 2012-02-21T08:53:14-0800 +2012-02-21T08:53:17-0800 199.233.217.249 2012-02-21T08:53:17-0800 +2012-02-21T08:52:41-0800 141.142.220.235 2012-02-21T08:52:41-0800 +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/utc-fmt b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/utc-fmt new file mode 100644 index 0000000000..8e8b7a0154 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-header/utc-fmt @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-15-51 +#fields ts uid id.orig_h id.orig_p id.resp_h +#types string string addr port addr +17 43 12 02 15 CjhGID4nQcgTWjvg4c 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +17 43 12 02 15 CCvvfg3TEfuqmmG4bh 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +17 43 12 02 15 CsRx2w45OKnoww6xl4 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +17 43 12 02 15 CRJuHdVW0XPVINV8a 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +17 42 12 02 15 CXWv6p3arKYeMETxOg 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +17 43 12 02 15 CPbrpk1qSsw6ESzHV4 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e +#close 2014-04-01-23-15-51 diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-optional/out b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-optional/out new file mode 100644 index 0000000000..9be9f2615a --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-optional/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2015-04-21T02:34:05+0000 CXWv6p3arKYeMETxOg 192.168.1.31 64889 192.168.1.32 88 TGS user/TEST.NET krbtgt/TEST.NET T - - 1970-01-01T00:00:00+0000 aes256-cts-hmac-sha1-96 T F - - - - diff --git a/auxil/zeek-aux/testing/Baseline/zeek-cut.time-optional/out2 b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-optional/out2 new file mode 100644 index 0000000000..ee19765790 --- /dev/null +++ b/auxil/zeek-aux/testing/Baseline/zeek-cut.time-optional/out2 @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +2015-04-21T02:34:05+0000 CXWv6p3arKYeMETxOg 192.168.1.31 64889 192.168.1.32 88 TGS user/TEST.NET krbtgt/TEST.NET T (unset_test) (unset_test) 1970-01-01T00:00:00+0000 aes256-cts-hmac-sha1-96 T F (unset_test) (unset_test) (unset_test) (unset_test) diff --git a/auxil/zeek-aux/testing/Logs/conn-tsv.log b/auxil/zeek-aux/testing/Logs/conn-tsv.log new file mode 100644 index 0000000000..32c41f47de --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/conn-tsv.log @@ -0,0 +1,6 @@ +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +1329843175.736107 CHUSyo3gjtvVLqrHGk 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CWMUpO2OZ5t5tLk6Hk 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CtII2N2AidtNJlD9f7 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843161.968492 CQyLvn3Dh4UDubXFRh 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +1329843197.783443 CuFJh714tTrtlGOxl7 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) diff --git a/auxil/zeek-aux/testing/Logs/conn.log b/auxil/zeek-aux/testing/Logs/conn.log new file mode 100644 index 0000000000..a614267726 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/conn.log @@ -0,0 +1,14 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-04-01-23-15-49 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1329843175.736107 CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Logs/conncomma.log b/auxil/zeek-aux/testing/Logs/conncomma.log new file mode 100644 index 0000000000..901eab0009 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/conncomma.log @@ -0,0 +1,14 @@ +#separator , +#set_separator,, +#empty_field,(empty) +#unset_field,- +#path,conn +#open,2014-06-30-16-10-54 +#fields,ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes,tunnel_parents +#types,time,string,addr,port,addr,port,enum,string,interval,count,count,string,bool,count,string,count,count,count,count,set[string] +1329843175.736107,CNbXUV0IZ29or3MK6,141.142.220.235,37604,199.233.217.249,56666,tcp,ftp-data,0.112432,0,342,SF,-,0,ShAdfFa,4,216,4,562,(empty) +1329843179.871641,CJ8woc3c6CfBLdiyp5,141.142.220.235,59378,199.233.217.249,56667,tcp,ftp-data,0.111218,0,77,SF,-,0,ShAdfFa,4,216,4,297,(empty) +1329843194.151526,CXlgj54ftP8Yc2GSnb,199.233.217.249,61920,141.142.220.235,33582,tcp,ftp-data,0.056211,342,0,SF,-,0,ShADaFf,5,614,3,164,(empty) +1329843161.968492,Czw8Gd1zEVn3Xz5x7i,141.142.220.235,50003,199.233.217.249,21,tcp,ftp,38.055625,180,3146,SF,-,0,ShAdDfFa,38,2164,25,4458,(empty) +1329843197.783443,Cys4aQ15qDqHzsIk3l,199.233.217.249,61918,141.142.220.235,37835,tcp,ftp-data,0.056005,77,0,SF,-,0,ShADaFf,5,349,3,164,(empty) +#close,2014-06-30-16-10-55 diff --git a/auxil/zeek-aux/testing/Logs/conntimelast.log b/auxil/zeek-aux/testing/Logs/conntimelast.log new file mode 100644 index 0000000000..b01f6a4adf --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/conntimelast.log @@ -0,0 +1,14 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conntimelast +#open 2014-04-01-23-15-49 +#fields uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents ts +#types string addr port addr port enum string interval count count string bool count string count count count count set[string] time +CjhGID4nQcgTWjvg4c 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) 1329843175.736107 +CCvvfg3TEfuqmmG4bh 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) 1329843179.871641 +CsRx2w45OKnoww6xl4 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) 1329843194.151526 +CRJuHdVW0XPVINV8a 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) 1329843197.783443 +CXWv6p3arKYeMETxOg 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) 1329843161.968492 +#close 2014-04-01-23-15-49 diff --git a/auxil/zeek-aux/testing/Logs/invalid-time.log b/auxil/zeek-aux/testing/Logs/invalid-time.log new file mode 100644 index 0000000000..50ac7393b2 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/invalid-time.log @@ -0,0 +1,7 @@ +#fields ts test +#types time string +hello a +-12345.123456 b +77777777777777777777 c + d +123456789 e diff --git a/auxil/zeek-aux/testing/Logs/missing-separator.log b/auxil/zeek-aux/testing/Logs/missing-separator.log new file mode 100644 index 0000000000..f4d2960f2d --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/missing-separator.log @@ -0,0 +1,3 @@ +#separator +#fields a +hi diff --git a/auxil/zeek-aux/testing/Logs/multiple-times.log b/auxil/zeek-aux/testing/Logs/multiple-times.log new file mode 100644 index 0000000000..79902de1bf --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/multiple-times.log @@ -0,0 +1,13 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-16-29 +#fields ts1 proto ts2 +#types time enum time +1329843175.736107 tcp 1329843185.736107 +1329843179.871641 udp 1329843179.982531 +1329843194.151526 tcp 1329843204.151526 +1329843204.987656 tcp 1329843255.123456 +#close 2014-04-01-23-16-29 diff --git a/auxil/zeek-aux/testing/Logs/null-separator.log b/auxil/zeek-aux/testing/Logs/null-separator.log new file mode 100644 index 0000000000..de4c693ff0 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/null-separator.log @@ -0,0 +1,3 @@ +#separator \x00 +#fields a +hi diff --git a/auxil/zeek-aux/testing/Logs/onecolumn.log b/auxil/zeek-aux/testing/Logs/onecolumn.log new file mode 100644 index 0000000000..01798435d0 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/onecolumn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path onecolumn +#open 2014-04-01-23-15-59 +#fields id.orig_h +#types addr +79.26.245.236 +#close 2014-04-01-23-15-59 diff --git a/auxil/zeek-aux/testing/Logs/test.log b/auxil/zeek-aux/testing/Logs/test.log new file mode 100644 index 0000000000..07107caed7 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/test.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-15-51 +#fields ts uid id.orig_h id.orig_p id.resp_h +#types time string addr port addr +1329327783.316897 CjhGID4nQcgTWjvg4c 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 CCvvfg3TEfuqmmG4bh 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 CsRx2w45OKnoww6xl4 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 CRJuHdVW0XPVINV8a 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 CXWv6p3arKYeMETxOg 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 CPbrpk1qSsw6ESzHV4 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e +#close 2014-04-01-23-15-51 diff --git a/auxil/zeek-aux/testing/Logs/time-optional.log b/auxil/zeek-aux/testing/Logs/time-optional.log new file mode 100644 index 0000000000..55d4b74dc4 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/time-optional.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path kerberos +#open 2015-04-21-19-22-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p request_type client service success error_msg from till cipher forwardable renewable client_cert_subject client_cert_fuid server_cert_subject server_cert_fuid +#types time string addr port addr port string string string bool string time time string bool bool string string string string +1429583645.478441 CXWv6p3arKYeMETxOg 192.168.1.31 64889 192.168.1.32 88 TGS user/TEST.NET krbtgt/TEST.NET T - - 0.000000 aes256-cts-hmac-sha1-96 T F - - - - +#close 2015-04-21-19-22-29 diff --git a/auxil/zeek-aux/testing/Logs/unset-field.log b/auxil/zeek-aux/testing/Logs/unset-field.log new file mode 100644 index 0000000000..230e08ed19 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/unset-field.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field (unset_test) +#path kerberos +#open 2015-04-21-19-22-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p request_type client service success error_msg from till cipher forwardable renewable client_cert_subject client_cert_fuid server_cert_subject server_cert_fuid +#types time string addr port addr port string string string bool string time time string bool bool string string string string +1429583645.478441 CXWv6p3arKYeMETxOg 192.168.1.31 64889 192.168.1.32 88 TGS user/TEST.NET krbtgt/TEST.NET T (unset_test) (unset_test) 0.000000 aes256-cts-hmac-sha1-96 T F (unset_test) (unset_test) (unset_test) (unset_test) +#close 2015-04-21-19-22-29 diff --git a/auxil/zeek-aux/testing/Logs/zerotime.log b/auxil/zeek-aux/testing/Logs/zerotime.log new file mode 100644 index 0000000000..acfbcb1c67 --- /dev/null +++ b/auxil/zeek-aux/testing/Logs/zerotime.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2014-04-01-23-15-51 +#fields ts uid id.orig_h id.orig_p id.resp_h +#types time string addr port addr +0000000000.000000 CjhGID4nQcgTWjvg4c 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49186 2001:470:4867:99::21 +1329327786.524332 CCvvfg3TEfuqmmG4bh 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49187 2001:470:4867:99::21 +1329327787.289095 CsRx2w45OKnoww6xl4 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49188 2001:470:4867:99::21 +1329327795.571921 CRJuHdVW0XPVINV8a 2001:470:4867:99::21 55785 2001:470:1f11:81f:c999:d94:aa7c:2e3e +1329327777.822004 CXWv6p3arKYeMETxOg 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 +1329327800.017649 CPbrpk1qSsw6ESzHV4 2001:470:4867:99::21 55647 2001:470:1f11:81f:c999:d94:aa7c:2e3e +#close 2014-04-01-23-15-51 diff --git a/auxil/zeek-aux/testing/Makefile b/auxil/zeek-aux/testing/Makefile new file mode 100644 index 0000000000..5c94538bb2 --- /dev/null +++ b/auxil/zeek-aux/testing/Makefile @@ -0,0 +1,19 @@ +BTEST=../../btest/btest +DIAG=diag.log + +all: cleanup btest-verbose + +# Showing all tests. +btest-verbose: + @test -x $(BTEST) && bt=$(BTEST) || bt=btest; $$bt -j -f $(DIAG) + +brief: cleanup btest-brief + +# Brief output showing only failed tests. +btest-brief: + @test -x $(BTEST) && bt=$(BTEST) || bt=btest; $$bt -j -b -f $(DIAG) + +cleanup: + @rm -f $(DIAG) + +.PHONY: all btest-verbose brief btest-brief cleanup diff --git a/auxil/zeek-aux/testing/Scripts/diff-time-zone b/auxil/zeek-aux/testing/Scripts/diff-time-zone new file mode 100755 index 0000000000..a0e16dc5a9 --- /dev/null +++ b/auxil/zeek-aux/testing/Scripts/diff-time-zone @@ -0,0 +1,8 @@ +#! /usr/bin/env bash +# +# Replace the time zone with "+0000". This is needed for any zeek-cut tests +# that convert UTC timestamps and show the time zone, because strftime +# with the "%z" format string behaves differently on OS X (it just always +# shows the local time zone instead of "+0000"). + +sed 's/[+-][0-9][0-9][0-9][0-9]/+0000/g' diff --git a/auxil/zeek-aux/testing/Scripts/git b/auxil/zeek-aux/testing/Scripts/git new file mode 100755 index 0000000000..36a648ee7b --- /dev/null +++ b/auxil/zeek-aux/testing/Scripts/git @@ -0,0 +1,16 @@ +#! /usr/bin/env bash +# Git wrapper script for use during testing. + +# Use original path so we find the system's installed git, not this wrapper. +PATH="$ORIGPATH" + +# Unsetting the following prevents git from reading ~/.gitconfig, +# including potential githooks. +HOME= +XDG_CONFIG_HOME= + +git -c init.defaultBranch=master \ + -c protocol.file.allow=always \ + -c user.name=zeektest \ + -c user.email=zeektest@zeek.org \ + "$@" diff --git a/auxil/zeek-aux/testing/Scripts/zeek-archiver-common.sh b/auxil/zeek-aux/testing/Scripts/zeek-archiver-common.sh new file mode 100644 index 0000000000..b11deabfd6 --- /dev/null +++ b/auxil/zeek-aux/testing/Scripts/zeek-archiver-common.sh @@ -0,0 +1,18 @@ +# Common functionality for zeek-archiver's tests, originally found in its own +# test.sh script that wasn't using btest. + +set -e +set -x + +function queue_dir { + mkdir -p queue + echo queue +} + +function archive_dir { + echo archive +} + +function archive_date_dir { + echo archive/2020-07-16 +} diff --git a/auxil/zeek-aux/testing/btest.cfg b/auxil/zeek-aux/testing/btest.cfg new file mode 100644 index 0000000000..c2d7d51bdf --- /dev/null +++ b/auxil/zeek-aux/testing/btest.cfg @@ -0,0 +1,15 @@ +[btest] +TestDirs = zeek-archiver zeek-cut update-changes +TmpDir = %(testbase)s/.tmp +BaselineDir = %(testbase)s/Baseline +IgnoreDirs = .svn CVS .tmp +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store + +[environment] +TZ=PST+8 +LC_ALL=C +ORIGPATH=%(default_path)s +PATH=%(testbase)s/Scripts:%(testbase)s/../../../build/auxil/zeek-aux/zeek-archiver:%(testbase)s/../build/zeek-archiver:%(testbase)s/../../../build/auxil/zeek-aux/zeek-cut:%(testbase)s/../build/zeek-cut:%(testbase)s/../../btest:%(testbase)s/../devel-tools:%(default_path)s +LOGS=%(testbase)s/Logs +TMPDIR=%(testbase)s/.tmp +SCRIPTS=%(testbase)s/Scripts diff --git a/auxil/zeek-aux/testing/update-changes/changes-up-to-date.sh b/auxil/zeek-aux/testing/update-changes/changes-up-to-date.sh new file mode 100644 index 0000000000..adae644de0 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/changes-up-to-date.sh @@ -0,0 +1,22 @@ +# Test update-changes -c. On an absent repo, this should fail. When there have +# not been commits since the last CHANGES update, it should succeed, and after +# subsequent commits it should fail again. +# +# @TEST-EXEC-FAIL: update-changes -c +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: update-changes -c +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC-FAIL: update-changes -c + +if [ ! -d .git ]; then + git init . + echo "Hello" >README + git add README + git commit -m 'init' + + echo "1.0.0" | update-changes -I +else + echo >>README + git add README + git commit -m 'readme update' +fi diff --git a/auxil/zeek-aux/testing/update-changes/initialize-from-prompt.sh b/auxil/zeek-aux/testing/update-changes/initialize-from-prompt.sh new file mode 100644 index 0000000000..74c9e1bda6 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/initialize-from-prompt.sh @@ -0,0 +1,15 @@ +# Test update-changes -I when no version information is present, and the user is +# prompted to provide one. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: test -f CHANGES +# @TEST-EXEC: grep -q '^1.0.0' CHANGES +# @TEST-EXEC: grep -q 'Starting CHANGES' CHANGES +# @TEST-EXEC: test $(git rev-list --count HEAD) -eq 2 + +git init . +echo "Hello" >README +git add README +git commit -m 'init' + +echo "1.0.0" | update-changes -I diff --git a/auxil/zeek-aux/testing/update-changes/initialize-from-tag.sh b/auxil/zeek-aux/testing/update-changes/initialize-from-tag.sh new file mode 100644 index 0000000000..421d8140ff --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/initialize-from-tag.sh @@ -0,0 +1,15 @@ +# Test update-changes -I when version information is available from git tags. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: test -f CHANGES +# @TEST-EXEC: grep -q '^1.0.0-1' CHANGES +# @TEST-EXEC: grep -q 'Starting CHANGES' CHANGES +# @TEST-EXEC: test $(git rev-list --count HEAD) -eq 2 + +git init . +echo "Hello" >README +git add README +git commit -m 'init' +git tag v1.0.0 + +update-changes -I diff --git a/auxil/zeek-aux/testing/update-changes/initialize-from-version.sh b/auxil/zeek-aux/testing/update-changes/initialize-from-version.sh new file mode 100644 index 0000000000..66d9cc2d91 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/initialize-from-version.sh @@ -0,0 +1,15 @@ +# Test update-changes -I when version information is in the VERSION file, which +# update-changes wants confirmation for. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: test -f CHANGES +# @TEST-EXEC: grep -q '^1.0.0' CHANGES +# @TEST-EXEC: grep -q 'Starting CHANGES' CHANGES +# @TEST-EXEC: test $(git rev-list --count HEAD) -eq 2 + +git init . +echo "1.0.0" >VERSION +git add VERSION +git commit -m 'init' + +echo y | update-changes -I diff --git a/auxil/zeek-aux/testing/update-changes/replace-version-in-c-header.sh b/auxil/zeek-aux/testing/update-changes/replace-version-in-c-header.sh new file mode 100644 index 0000000000..0639f010ef --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/replace-version-in-c-header.sh @@ -0,0 +1,34 @@ +# Verifies that update-changes correctly updates version strings in C header +# files, when .update-changes.cfg instructs it. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: btest-diff header.h + +@TEST-START-FILE .update-changes.cfg +function new_version_hook() { + local version=$1 + replace_version_in_c_header header.h $version +} +@TEST-END-FILE + +git init + +cat >header.h <>header.h +git add header.h +git commit -m 'update' + +# Suppress input prompts: +export EDITOR=cat +printf '\n' | update-changes diff --git a/auxil/zeek-aux/testing/update-changes/replace-version-in-python-package.sh b/auxil/zeek-aux/testing/update-changes/replace-version-in-python-package.sh new file mode 100644 index 0000000000..9479097a89 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/replace-version-in-python-package.sh @@ -0,0 +1,36 @@ +# Verifies that update-changes correctly updates __version__ values as given in +# package-level __init__.py files, when .update-changes.cfg instructs it. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: btest-diff __init__.py + +@TEST-START-FILE .update-changes.cfg +function new_version_hook() { + local version=$1 + replace_version_in_python_package __init__.py $version +} +@TEST-END-FILE + +git init + +cat >__init__.py <>__init__.py +git add __init__.py +git commit -m 'update' + +# Suppress input prompts: +export EDITOR=cat +printf '\n' | update-changes diff --git a/auxil/zeek-aux/testing/update-changes/replace-version-in-rst.sh b/auxil/zeek-aux/testing/update-changes/replace-version-in-rst.sh new file mode 100644 index 0000000000..3c38bc8db4 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/replace-version-in-rst.sh @@ -0,0 +1,36 @@ +# Verifies that update-changes correctly updates version strings in +# ReST docs, when .update-changes.cfg instructs it. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: btest-diff test.rst + +@TEST-START-FILE .update-changes.cfg +function new_version_hook() { + local version=$1 + replace_version_in_rst test.rst $version +} +@TEST-END-FILE + +git init + +cat >test.rst <>test.rst +git add test.rst +git commit -m 'update' + +# Suppress input prompts: +export EDITOR=cat +printf '\n' | update-changes diff --git a/auxil/zeek-aux/testing/update-changes/replace-version-in-script.sh b/auxil/zeek-aux/testing/update-changes/replace-version-in-script.sh new file mode 100644 index 0000000000..4cc35fec8c --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/replace-version-in-script.sh @@ -0,0 +1,37 @@ +# Verifies that update-changes correctly updates version strings in +# shell/Python/etc scripts, when .update-changes.cfg instructs it. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: btest-diff test.pl + +@TEST-START-FILE .update-changes.cfg +function new_version_hook() { + local version=$1 + replace_version_in_script test.pl $version +} +@TEST-END-FILE + +git init + +cat >test.pl <>test.pl +git add test.pl +git commit -m 'update' + +# Suppress input prompts: +export EDITOR=cat +printf '\n' | update-changes diff --git a/auxil/zeek-aux/testing/update-changes/replace-version-in-setup-py.sh b/auxil/zeek-aux/testing/update-changes/replace-version-in-setup-py.sh new file mode 100644 index 0000000000..841ee9ac2f --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/replace-version-in-setup-py.sh @@ -0,0 +1,37 @@ +# Verifies that update-changes correctly updates version strings in setup.py +# files to Python-style versions, when .update-changes.cfg instructs it. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: btest-diff setup.py + +@TEST-START-FILE .update-changes.cfg +function new_version_hook() { + local version=$1 + replace_version_in_setup_py setup.py $version +} +@TEST-END-FILE + +git init + +cat >setup.py <>setup.py +git add setup.py +git commit -m 'update' + +# Suppress input prompts: +export EDITOR=cat +printf '\n' | update-changes diff --git a/auxil/zeek-aux/testing/update-changes/tag-dev-point-release-ahead-of-origin.sh b/auxil/zeek-aux/testing/update-changes/tag-dev-point-release-ahead-of-origin.sh new file mode 100644 index 0000000000..a3c7a28bd4 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/tag-dev-point-release-ahead-of-origin.sh @@ -0,0 +1,39 @@ +# Like tag-point-release-ahead-of-origin, but on top of a "-dev" git tag +# to test the underlying -r regex. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: cd clone && git describe --exact-match HEAD | grep -q v1.0.1 +# @TEST-EXEC: cd clone && git log --format=%B -n 1 HEAD | grep -q 'This is 1.0.1' +# @TEST-EXEC: cd clone && head -1 CHANGES | grep -q '^1.0.1' +# @TEST-EXEC: cd clone && test $(git rev-list --count HEAD) -eq 4 + +( + mkdir origin && cd origin + + git init + echo "Hello" >README + git add README + git commit -m 'init' + git tag v1.0.0-dev +) + +# We need an origin to control update-change's augment-vs-new-commit logic. +git clone origin clone + +( + cd clone + + update-changes -I + + echo ... >>README + git add README + git commit -m 'readme update' + + echo "Meet v1.0.1" >>README + git add README + git commit -m "This is 1.0.1" + + # Suppress input prompts: + export EDITOR=cat + printf '\n' | update-changes -r +) diff --git a/auxil/zeek-aux/testing/update-changes/tag-point-release-ahead-of-origin.sh b/auxil/zeek-aux/testing/update-changes/tag-point-release-ahead-of-origin.sh new file mode 100644 index 0000000000..7405f4b7d6 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/tag-point-release-ahead-of-origin.sh @@ -0,0 +1,39 @@ +# Test update-changes -r when the repo is cloned from an origin. With commits +# ahead of CHANGES, the release commit should be augmented onto the last. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: cd clone && git describe --exact-match HEAD | grep -q v1.0.1 +# @TEST-EXEC: cd clone && git log --format=%B -n 1 HEAD | grep -q 'This is 1.0.1' +# @TEST-EXEC: cd clone && head -1 CHANGES | grep -q '^1.0.1' +# @TEST-EXEC: cd clone && test $(git rev-list --count HEAD) -eq 4 + +( + mkdir origin && cd origin + + git init + echo "Hello" >README + git add README + git commit -m 'init' + git tag v1.0.0 +) + +# We need an origin to control update-change's augment-vs-new-commit logic. +git clone origin clone + +( + cd clone + + update-changes -I + + echo ... >>README + git add README + git commit -m 'readme update' + + echo "Meet v1.0.1" >>README + git add README + git commit -m "This is 1.0.1" + + # Suppress input prompts: + export EDITOR=cat + printf '\n' | update-changes -r +) diff --git a/auxil/zeek-aux/testing/update-changes/tag-point-release-at-origin.sh b/auxil/zeek-aux/testing/update-changes/tag-point-release-at-origin.sh new file mode 100644 index 0000000000..1f5e66735c --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/tag-point-release-at-origin.sh @@ -0,0 +1,35 @@ +# Test update-changes -r when the repo is up to date with its origin. In that +# case, the release commit should become a new one. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: cd clone && git describe --exact-match HEAD | grep -q v1.0.1 +# @TEST-EXEC: cd clone && git log --format=%B -n 1 HEAD | grep -q 'Updating CHANGES and VERSION' +# @TEST-EXEC: cd clone && head -1 CHANGES | grep -q '^1.0.1' +# @TEST-EXEC: cd clone && test $(git rev-list --count HEAD) -eq 4 + +( + mkdir origin && cd origin + + git init + echo "Hello" >README + git add README + git commit -m 'init' + git tag v1.0.0 + + update-changes -I + + echo ... >>README + git add README + git commit -m 'readme update' +) + +# We need an origin to control update-change's augment-vs-new-commit logic. +git clone origin clone + +( + cd clone + + # Suppress input prompts: + export EDITOR=cat + printf '\n' | update-changes -r +) diff --git a/auxil/zeek-aux/testing/update-changes/tag-release-ahead-of-origin-new-commit.sh b/auxil/zeek-aux/testing/update-changes/tag-release-ahead-of-origin-new-commit.sh new file mode 100644 index 0000000000..806b83d198 --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/tag-release-ahead-of-origin-new-commit.sh @@ -0,0 +1,40 @@ +# Test update-changes -R when the repo is cloned from an origin. With commits +# ahead of CHANGES the release commit would be augmented onto the last, but +# this also uses -n to create a new one. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: cd clone && git describe --exact-match HEAD | grep -q v2.0.0 +# @TEST-EXEC: cd clone && git log --format=%B -n 1 HEAD | grep -q 'Updating CHANGES and VERSION' +# @TEST-EXEC: cd clone && head -1 CHANGES | grep -q '^2.0.0' +# @TEST-EXEC: cd clone && test $(git rev-list --count HEAD) -eq 5 + +( + mkdir origin && cd origin + + git init + echo "Hello" >README + git add README + git commit -m 'init' + git tag v1.0.0 +) + +# We need an origin to control update-change's augment-vs-new-commit logic. +git clone origin clone + +( + cd clone + + update-changes -I + + echo ... >>README + git add README + git commit -m 'readme update' + + echo "Meet v2.0.0" >>README + git add README + git commit -m "This is 2.0.0" + + # Suppress input prompts: + export EDITOR=cat + printf '\n' | update-changes -R v2.0.0 -n +) diff --git a/auxil/zeek-aux/testing/update-changes/tag-release-ahead-of-origin.sh b/auxil/zeek-aux/testing/update-changes/tag-release-ahead-of-origin.sh new file mode 100644 index 0000000000..5e741f025b --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/tag-release-ahead-of-origin.sh @@ -0,0 +1,39 @@ +# Test update-changes -R when the repo is cloned from an origin. With commits +# ahead of CHANGES the release commit should be augmented onto the last. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: cd clone && git describe --exact-match HEAD | grep -q v2.0.0 +# @TEST-EXEC: cd clone && git log --format=%B -n 1 HEAD | grep -q 'This is 2.0.0' +# @TEST-EXEC: cd clone && head -1 CHANGES | grep -q '^2.0.0' +# @TEST-EXEC: cd clone && test $(git rev-list --count HEAD) -eq 4 + +( + mkdir origin && cd origin + + git init + echo "Hello" >README + git add README + git commit -m 'init' + git tag v1.0.0 +) + +# We need an origin to control update-change's augment-vs-new-commit logic. +git clone origin clone + +( + cd clone + + update-changes -I + + echo ... >>README + git add README + git commit -m 'readme update' + + echo "Meet v2.0.0" >>README + git add README + git commit -m "This is 2.0.0" + + # Suppress input prompts: + export EDITOR=cat + printf '\n' | update-changes -R v2.0.0 +) diff --git a/auxil/zeek-aux/testing/update-changes/tag-release-at-origin.sh b/auxil/zeek-aux/testing/update-changes/tag-release-at-origin.sh new file mode 100644 index 0000000000..2456aa64dc --- /dev/null +++ b/auxil/zeek-aux/testing/update-changes/tag-release-at-origin.sh @@ -0,0 +1,35 @@ +# Test update-changes -R when the repo is up to date with its origin. In that +# case, the release commit should become a new one. +# +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: cd clone && git describe --exact-match HEAD | grep -q v2.0.0 +# @TEST-EXEC: cd clone && git log --format=%B -n 1 HEAD | grep -q 'Updating CHANGES and VERSION' +# @TEST-EXEC: cd clone && head -1 CHANGES | grep -q '^2.0.0' +# @TEST-EXEC: cd clone && test $(git rev-list --count HEAD) -eq 4 + +( + mkdir origin && cd origin + + git init + echo "Hello" >README + git add README + git commit -m 'init' + git tag v1.0.0 + + update-changes -I + + echo ... >>README + git add README + git commit -m 'readme update' +) + +# We need an origin to control update-change's augment-vs-new-commit logic. +git clone origin clone + +( + cd clone + + # Suppress input prompts: + export EDITOR=cat + printf '\n' | update-changes -R v2.0.0 +) diff --git a/auxil/zeek-aux/testing/zeek-archiver/command-injection-filenames.test b/auxil/zeek-aux/testing/zeek-archiver/command-injection-filenames.test new file mode 100644 index 0000000000..79d60e0e82 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/command-injection-filenames.test @@ -0,0 +1,12 @@ +# Verify that commands injected via filenames do not execute. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in='test;uptime;__2020-07-16-09-43-10__2020-07-16-09-43-10__.log' +log_out='test;uptime;.09:43:10-09:43:10.log' + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/custom-delimiter.test b/auxil/zeek-aux/testing/zeek-archiver/custom-delimiter.test new file mode 100644 index 0000000000..3ac3e5847e --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/custom-delimiter.test @@ -0,0 +1,12 @@ +# Use a custom delimiter in the resulting logs. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test_2020-07-16-09-43-10_2020-07-16-09-43-10_.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v -d _ "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <$(archive_date_dir)/${log_out}.gz)" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/custom-timestamp.test b/auxil/zeek-aux/testing/zeek-archiver/custom-timestamp.test new file mode 100644 index 0000000000..bf03e60b08 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/custom-timestamp.test @@ -0,0 +1,12 @@ +# Use a customized timestamp in output logs. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16_09-43-10__2020-07-16_09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v --time-fmt %Y-%m-%d_%H-%M-%S "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/default.test b/auxil/zeek-aux/testing/zeek-archiver/default.test new file mode 100644 index 0000000000..ce7cb7a24c --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/default.test @@ -0,0 +1,12 @@ +# Default behavior: compress the log. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/disable-compression.test b/auxil/zeek-aux/testing/zeek-archiver/disable-compression.test new file mode 100644 index 0000000000..eca7d5c327 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/disable-compression.test @@ -0,0 +1,12 @@ +# Don't compress the logs. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v --compress="" "$(queue_dir)" "$(archive_dir)" + +test "$(cat "$(archive_date_dir)/${log_out}")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/extra-compression-args.test b/auxil/zeek-aux/testing/zeek-archiver/extra-compression-args.test new file mode 100644 index 0000000000..65781240b4 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/extra-compression-args.test @@ -0,0 +1,12 @@ +# Verify that passing extra parameters to the compression stage works. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v --compress 'gz,gzip -9' "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/failing-compress.test b/auxil/zeek-aux/testing/zeek-archiver/failing-compress.test new file mode 100644 index 0000000000..259b6f64b1 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/failing-compress.test @@ -0,0 +1,15 @@ +# Verify the source file still exists and the destination wasn't created +# (or removed) when the compression command fails. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v --compress=false,/bin/false "$(queue_dir)" "$(archive_dir)" + +if [[ ! -e "$(queue_dir)/${log_in}" ]] || [[ -e "$(archive_date_dir)/${log_out}.false" ]] ; then + exit 1 +fi diff --git a/auxil/zeek-aux/testing/zeek-archiver/metadata-addl.test b/auxil/zeek-aux/testing/zeek-archiver/metadata-addl.test new file mode 100644 index 0000000000..524409f02e --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/metadata-addl.test @@ -0,0 +1,12 @@ +# Verify log name metadata behavior: check that additional pid metadata gets dropped. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-test,pid=4711__.log +log_out=test.09:43:10-09:43:10-logger-test.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/metadata-empty.test b/auxil/zeek-aux/testing/zeek-archiver/metadata-empty.test new file mode 100644 index 0000000000..ff9860d605 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/metadata-empty.test @@ -0,0 +1,12 @@ +# Verify log name metadata behavior: empty metadata is acceptable. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10____.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/metadata-invalid.test b/auxil/zeek-aux/testing/zeek-archiver/metadata-invalid.test new file mode 100644 index 0000000000..11d2dad26f --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/metadata-invalid.test @@ -0,0 +1,21 @@ +# Verify log name metadata behavior: invalid metadata causes skipping of archival. +# @TEST-EXEC: bash %INPUT + +@TEST-START-FILE run.sh +. "$SCRIPTS/zeek-archiver-common.sh" + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" +test -f "$(queue_dir)/${log_in}" +@TEST-END-FILE + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix,invalid=4711__.log +. run.sh + +# @TEST-START-NEXT +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-test,__.log +. run.sh + +# @TEST-START-NEXT +log_in="test__2020-07-16-09-43-10__2020-07-16-09-43-10__ __.log" +. run.sh diff --git a/auxil/zeek-aux/testing/zeek-archiver/metadata.test b/auxil/zeek-aux/testing/zeek-archiver/metadata.test new file mode 100644 index 0000000000..8add2e390f --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/metadata.test @@ -0,0 +1,12 @@ +# Verify log name metadata behavior: check that suffixes get applied correctly. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-test__.log +log_out=test.09:43:10-09:43:10-logger-test.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/pre-compression-logs.test b/auxil/zeek-aux/testing/zeek-archiver/pre-compression-logs.test new file mode 100644 index 0000000000..6a9ec6a9be --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/pre-compression-logs.test @@ -0,0 +1,13 @@ +# Detect pre-compressed logs. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +gzip "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +test "$(gunzip <"$(archive_date_dir)/${log_out}.gz")" == "hello" diff --git a/auxil/zeek-aux/testing/zeek-archiver/umask.test b/auxil/zeek-aux/testing/zeek-archiver/umask.test new file mode 100644 index 0000000000..ef64629de3 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-archiver/umask.test @@ -0,0 +1,36 @@ +# @TEST-DOC: Check file permissions with different umask settings. +# @TEST-EXEC: bash %INPUT + +. "$SCRIPTS/zeek-archiver-common.sh" + +umask 0002 + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +dir_perms=$(ls -ld $(archive_date_dir) | cut -c -10) +file_perms=$(ls -l $(archive_date_dir)/${log_out}.gz | cut -c -10) + +test "${dir_perms}" == "drwxrwxr-x" || exit 1 +test "${file_perms}" == "-rw-rw-r--" || exit 1 + +# @TEST-START-NEXT + +. "$SCRIPTS/zeek-archiver-common.sh" + +umask 0077 + +log_in=test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log +log_out=test.09:43:10-09:43:10.log + +echo hello > "$(queue_dir)/${log_in}" +zeek-archiver -1 -v "$(queue_dir)" "$(archive_dir)" + +dir_perms=$(ls -ld $(archive_date_dir) | cut -c -10) +file_perms=$(ls -l $(archive_date_dir)/${log_out}.gz | cut -c -10) + +test "${dir_perms}" == "drwx------" || exit 1 +test "${file_perms}" == "-rw-------" || exit 1 diff --git a/auxil/zeek-aux/testing/zeek-cut/bad-logs.test b/auxil/zeek-aux/testing/zeek-cut/bad-logs.test new file mode 100644 index 0000000000..00bb132e4c --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/bad-logs.test @@ -0,0 +1,30 @@ +# Test zeek-cut with invalid log files. +# + +# The "#separator" log header line is missing. +# @TEST-EXEC: grep -v '^#separator' $LOGS/conn.log | zeek-cut uid proto > missing-sep-header +# @TEST-EXEC: btest-diff missing-sep-header + +# The "#fields" log header line is missing (show header). +# @TEST-EXEC-FAIL: grep -v '^#fields' $LOGS/conn.log | zeek-cut -C 2> missing-fields-header +# @TEST-EXEC: btest-diff missing-fields-header + +# All log header lines are missing (show header). +# @TEST-EXEC: grep -v '^#' $LOGS/conn.log | zeek-cut -c > no-header-show +# @TEST-EXEC: btest-diff no-header-show + +# All log header lines are missing (select column to show). +# @TEST-EXEC: grep -v '^#' $LOGS/conn.log | zeek-cut uid > no-header-column +# @TEST-EXEC: btest-diff no-header-column + +# All log header lines are missing (select column to not show). +# @TEST-EXEC: grep -v '^#' $LOGS/conn.log | zeek-cut -n uid > no-header-not-column +# @TEST-EXEC: btest-diff no-header-not-column + +# Separator is missing +# @TEST-EXEC-FAIL: cat $LOGS/missing-separator.log | zeek-cut 2> missing-separator +# @TEST-EXEC: btest-diff missing-separator + +# Separator is \x00 +# @TEST-EXEC-FAIL: cat $LOGS/null-separator.log | zeek-cut 2> null-separator +# @TEST-EXEC: btest-diff null-separator diff --git a/auxil/zeek-aux/testing/zeek-cut/columns.test b/auxil/zeek-aux/testing/zeek-cut/columns.test new file mode 100644 index 0000000000..031906d44f --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/columns.test @@ -0,0 +1,34 @@ +# Test zeek-cut with column names, but no other options. +# + +# Select one column. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut uid > one +# @TEST-EXEC: btest-diff one + +# Log file has only one column. +# @TEST-EXEC: cat $LOGS/onecolumn.log | zeek-cut id.orig_h > only +# @TEST-EXEC: btest-diff only + +# Swap the order of two columns. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut proto uid > swap-order +# @TEST-EXEC: btest-diff swap-order + +# Specify all columns in the log file. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut ts uid id.orig_h id.orig_p id.resp_h > all +# @TEST-EXEC: btest-diff all + +# Log files use different field separators. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conncomma.log | zeek-cut proto uid > nondefault-separator +# @TEST-EXEC: btest-diff nondefault-separator + +# Column order changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut ts uid > different-col-order +# @TEST-EXEC: btest-diff different-col-order + +# Specify a column name that exists only in the first log file. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut uid proto > one-nonexistent-1 +# @TEST-EXEC: btest-diff one-nonexistent-1 + +# Specify a column name that exists only in the second log file. +# @TEST-EXEC: cat $LOGS/test.log $LOGS/conn.log | zeek-cut uid proto > one-nonexistent-2 +# @TEST-EXEC: btest-diff one-nonexistent-2 diff --git a/auxil/zeek-aux/testing/zeek-cut/help.test b/auxil/zeek-aux/testing/zeek-cut/help.test new file mode 100644 index 0000000000..d605efe81c --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/help.test @@ -0,0 +1,4 @@ +# Test zeek-cut with -h option. +# +# @TEST-EXEC-FAIL: zeek-cut -h > show-help +# @TEST-EXEC: btest-diff show-help diff --git a/auxil/zeek-aux/testing/zeek-cut/minimal-view.test b/auxil/zeek-aux/testing/zeek-cut/minimal-view.test new file mode 100644 index 0000000000..6f434570c8 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/minimal-view.test @@ -0,0 +1,50 @@ +# Test zeek-cut with -m and -M options. +# + +# Show first header in minimal view for one log file. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -m > one-all-nocols +# @TEST-EXEC: btest-diff one-all-nocols + +# Show first header in minimal view for two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -m uid ts > two-1st-withcols +# @TEST-EXEC: btest-diff two-1st-withcols + +# Show all headers in minimal view for two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -M uid ts > two-all-withcols +# @TEST-EXEC: btest-diff two-all-withcols +# +# Show first header in minimal view for two log files with custom field separator. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -F, -m uid ts > two-1st-withcols-custom-sep +# @TEST-EXEC: btest-diff two-1st-withcols-custom-sep + +# Show all headers in minimal view for two log files with custom field separator. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -F, -M uid ts > two-all-withcols-custom-sep +# @TEST-EXEC: btest-diff two-all-withcols-custom-sep + +# Show all headers in minimal view for two logs (-M overrides -m). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -m -M uid ts > both-m-opts-all +# @TEST-EXEC: btest-diff both-m-opts-all + +# Show first header in minimal view for two logs (-m overrides -M). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -M -m uid ts > both-m-opts-one +# @TEST-EXEC: btest-diff both-m-opts-one + +# Show first header in minimal view for two logs (-m overrides -C). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -C -m uid ts > both-c-m-opts-m +# @TEST-EXEC: btest-diff both-c-m-opts-m + +# Show all headers for two logs (-c overrides -M). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -M -c uid ts > both-m-c-opts-c +# @TEST-EXEC: btest-diff both-m-c-opts-c + +# Log files use different field separators. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conncomma.log | zeek-cut -M proto uid > nondefault-separator +# @TEST-EXEC: btest-diff nondefault-separator + +# Column order changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -M > different-col-order +# @TEST-EXEC: btest-diff different-col-order + +# Column order changes between two log files (and don't show all columns). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -M ts uid > different-col-order-some +# @TEST-EXEC: btest-diff different-col-order-some diff --git a/auxil/zeek-aux/testing/zeek-cut/no-options.test b/auxil/zeek-aux/testing/zeek-cut/no-options.test new file mode 100644 index 0000000000..a2a641fdd9 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/no-options.test @@ -0,0 +1,9 @@ +# Test zeek-cut without any command-line options. +# + +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut > no-opts +# @TEST-EXEC: btest-diff no-opts + +# Log file in "tsv" (tab-separated-values) format. +# @TEST-EXEC: cat $LOGS/conn-tsv.log | zeek-cut > tsv +# @TEST-EXEC: btest-diff tsv diff --git a/auxil/zeek-aux/testing/zeek-cut/not-columns.test b/auxil/zeek-aux/testing/zeek-cut/not-columns.test new file mode 100644 index 0000000000..e7356e9d5d --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/not-columns.test @@ -0,0 +1,38 @@ +# Test zeek-cut with the -n option. +# + +# Show all columns but one. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -n uid > not-one +# @TEST-EXEC: btest-diff not-one + +# Show all columns but two (and specify them in swapped order). +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -n id.resp_h uid > not-two-swapped +# @TEST-EXEC: btest-diff not-two-swapped + +# Show all columns but one, but specify the same column twice. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -n uid uid > not-one-twice +# @TEST-EXEC: btest-diff not-one-twice + +# Column order changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -n ts uid > different-col-order +# @TEST-EXEC: btest-diff different-col-order + +# Don't show any columns. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -n ts uid id.orig_h id.orig_p id.resp_h > not-all +# @TEST-EXEC: btest-diff not-all + +# Don't show the only column in the log. +# @TEST-EXEC: cat $LOGS/onecolumn.log | zeek-cut -n id.orig_h > not-only +# @TEST-EXEC: btest-diff not-only + +# Don't exclude any columns. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -n > not-none +# @TEST-EXEC: btest-diff not-none + +# Show all columns but one, and also specify a nonexistent column name. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -n uid proto > not-one-nonexistent +# @TEST-EXEC: btest-diff not-one-nonexistent + +# Show that the -n option also affects the header. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -c -n uid > not-one-show-header +# @TEST-EXEC: btest-diff not-one-show-header diff --git a/auxil/zeek-aux/testing/zeek-cut/ofs.test b/auxil/zeek-aux/testing/zeek-cut/ofs.test new file mode 100644 index 0000000000..48914ade0c --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/ofs.test @@ -0,0 +1,18 @@ +# Test zeek-cut with -F option. +# + +# Use a different field separator on two columns. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -F , uid ts > some-columns +# @TEST-EXEC: btest-diff some-columns + +# Show that -F option is used even when field separator in log files changes. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conncomma.log | zeek-cut -F : proto uid > nondefault-separator +# @TEST-EXEC: btest-diff nondefault-separator + +# Field separator is not relevant because log has only one column. +# @TEST-EXEC: cat $LOGS/onecolumn.log | zeek-cut -F , > only-column +# @TEST-EXEC: btest-diff only-column + +# Show header and use a different field separator. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -F . -c > show-header +# @TEST-EXEC: btest-diff show-header diff --git a/auxil/zeek-aux/testing/zeek-cut/show-header.test b/auxil/zeek-aux/testing/zeek-cut/show-header.test new file mode 100644 index 0000000000..e0098e5c83 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/show-header.test @@ -0,0 +1,38 @@ +# Test zeek-cut with -c and -C options. +# + +# Show first header for one log file. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -c > one-1st-nocols +# @TEST-EXEC: btest-diff one-1st-nocols + +# Show all headers for one log file. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -C > one-all-nocols +# @TEST-EXEC: btest-diff one-all-nocols + +# Show first header for two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -c uid ts > two-1st-withcols +# @TEST-EXEC: btest-diff two-1st-withcols + +# Show all headers for two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -C uid ts > two-all-withcols +# @TEST-EXEC: btest-diff two-all-withcols + +# Show all headers for two logs (-C overrides -c). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -c -C uid ts > both-c-opts-all +# @TEST-EXEC: btest-diff both-c-opts-all + +# Show first header for two logs (-c overrides -C). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/test.log | zeek-cut -C -c uid ts > both-c-opts-one +# @TEST-EXEC: btest-diff both-c-opts-one + +# Log files use different field separators. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conncomma.log | zeek-cut -C proto uid > nondefault-separator +# @TEST-EXEC: btest-diff nondefault-separator + +# Column order changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -C > different-col-order +# @TEST-EXEC: btest-diff different-col-order + +# Column order changes between two log files (and don't show all columns). +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -C ts uid > different-col-order-some +# @TEST-EXEC: btest-diff different-col-order-some diff --git a/auxil/zeek-aux/testing/zeek-cut/time-bad-logs.test b/auxil/zeek-aux/testing/zeek-cut/time-bad-logs.test new file mode 100644 index 0000000000..47ffc55df2 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/time-bad-logs.test @@ -0,0 +1,15 @@ +# Test zeek-cut time conversion with invalid log files. +# + +# The "#types" log header line is missing. +# @TEST-EXEC-FAIL: grep -v '^#types' $LOGS/conn.log | zeek-cut -d 2> missing-types-header +# @TEST-EXEC: btest-diff missing-types-header + +# All log header lines are missing. +# @TEST-EXEC: grep -v '^#' $LOGS/conn.log | zeek-cut -d > no-header-time +# @TEST-EXEC: btest-diff no-header-time + +# Time field is invalid or out of range +# @TEST-EXEC: cat $LOGS/invalid-time.log | zeek-cut -d > invalid-time-field 2> invalid-time-field.err +# @TEST-EXEC: btest-diff invalid-time-field +# @TEST-EXEC: btest-diff invalid-time-field.err diff --git a/auxil/zeek-aux/testing/zeek-cut/time-conversion.test b/auxil/zeek-aux/testing/zeek-cut/time-conversion.test new file mode 100644 index 0000000000..3878e65dfb --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/time-conversion.test @@ -0,0 +1,42 @@ +# Test zeek-cut with time conversion options (-d/-u). +# + +# Test -d option. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -d > local +# @TEST-EXEC: btest-diff local + +# Test -u option. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -u > utc +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-time-zone btest-diff utc + +# Test that -u can override -d. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -d -u > both-1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-time-zone btest-diff both-1 + +# Test that -d can override -u. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -u -d > both-2 +# @TEST-EXEC: btest-diff both-2 + +# First time value is zero. +# @TEST-EXEC: cat $LOGS/zerotime.log | zeek-cut -u > zero +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-time-zone btest-diff zero + +# File has timestamp, but not in the first column. +# @TEST-EXEC: cat $LOGS/conntimelast.log | zeek-cut -d > time-last-col +# @TEST-EXEC: btest-diff time-last-col + +# File has more than one time column, and some timestamps are duplicates. +# @TEST-EXEC: cat $LOGS/multiple-times.log | zeek-cut -d > multiple-times +# @TEST-EXEC: btest-diff multiple-times + +# Column order changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -d > different-col-order +# @TEST-EXEC: btest-diff different-col-order + +# File has no timestamps. +# @TEST-EXEC: cat $LOGS/onecolumn.log | zeek-cut -d > no-timestamps +# @TEST-EXEC: btest-diff no-timestamps + +# Input is missing the log header lines. +# @TEST-EXEC: grep -v '^#' $LOGS/conn.log | zeek-cut -d > missing-header +# @TEST-EXEC: btest-diff missing-header diff --git a/auxil/zeek-aux/testing/zeek-cut/time-fmt-env.test b/auxil/zeek-aux/testing/zeek-cut/time-fmt-env.test new file mode 100644 index 0000000000..57d959bf69 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/time-fmt-env.test @@ -0,0 +1,18 @@ +# Test zeek-cut with ZEEK_CUT_TIMEFMT environment variable. +# + +# Output local time with format from environment. +# @TEST-EXEC: cat $LOGS/conn.log | ZEEK_CUT_TIMEFMT="%H%M_%y%m%d" zeek-cut -d > env-local +# @TEST-EXEC: btest-diff env-local + +# Output UTC with format from environment. +# @TEST-EXEC: cat $LOGS/conn.log | ZEEK_CUT_TIMEFMT="%H%M_%y%m%d" zeek-cut -u > env-utc +# @TEST-EXEC: btest-diff env-utc + +# The "-D" option overrides environment variable. +# @TEST-EXEC: cat $LOGS/conn.log | ZEEK_CUT_TIMEFMT="%H%M_%y%m%d" zeek-cut -D "%H %M %y %m %d" > env-local-fmt +# @TEST-EXEC: btest-diff env-local-fmt + +# The "-U" option overrides environment variable. +# @TEST-EXEC: cat $LOGS/conn.log | ZEEK_CUT_TIMEFMT="%H%M_%y%m%d" zeek-cut -U "%H %M %y %m %d" > env-utc-fmt +# @TEST-EXEC: btest-diff env-utc-fmt diff --git a/auxil/zeek-aux/testing/zeek-cut/time-fmt.test b/auxil/zeek-aux/testing/zeek-cut/time-fmt.test new file mode 100644 index 0000000000..e12235bf4d --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/time-fmt.test @@ -0,0 +1,26 @@ +# Test zeek-cut with time conversion options (-D/-U). +# + +# Test -D option. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -D "%H %M %y %m %d" > local-fmt +# @TEST-EXEC: btest-diff local-fmt + +# Test -U option. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -U "%H %M %y %m %d" > utc-fmt +# @TEST-EXEC: btest-diff utc-fmt + +# Output local time (-D overrides -u). +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -u -D "%H %M %y %m %d" > utc-local-fmt +# @TEST-EXEC: btest-diff utc-local-fmt + +# Output UTC time, but with format string from -D. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -D "%H %M %y %m %d" -u > local-fmt-utc +# @TEST-EXEC: btest-diff local-fmt-utc + +# Output UTC time (-U overrides -d). +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -d -U "%H %M %y %m %d" > local-utc-fmt +# @TEST-EXEC: btest-diff local-utc-fmt + +# Output local time, but with format string from -U. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -U "%H %M %y %m %d" -d > utc-fmt-local +# @TEST-EXEC: btest-diff utc-fmt-local diff --git a/auxil/zeek-aux/testing/zeek-cut/time-header.test b/auxil/zeek-aux/testing/zeek-cut/time-header.test new file mode 100644 index 0000000000..d7ac7688e6 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/time-header.test @@ -0,0 +1,30 @@ +# Test zeek-cut with both time conversion and show header options. +# + +# Time format string does not affect the header timestamps. +# @TEST-EXEC: cat $LOGS/test.log | zeek-cut -C -U "%H %M %y %m %d" > utc-fmt +# @TEST-EXEC: btest-diff utc-fmt + +# Log file uses non-default field separator. +# @TEST-EXEC: cat $LOGS/conncomma.log | zeek-cut -C -d proto ts id.orig_h > nondefault-separator +# @TEST-EXEC: btest-diff nondefault-separator + +# Field separator changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conncomma.log | zeek-cut -C -d proto ts id.orig_h > different-field-separator +# @TEST-EXEC: btest-diff different-field-separator + +# Column order changes between two log files. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/conntimelast.log | zeek-cut -C -d proto ts id.orig_h > different-col-order +# @TEST-EXEC: btest-diff different-col-order + +# One file has timestamps and one file does not. +# @TEST-EXEC: cat $LOGS/conn.log $LOGS/onecolumn.log | zeek-cut -C -d ts id.orig_h > missing-ts-one-file +# @TEST-EXEC: btest-diff missing-ts-one-file + +# Specify ts twice. +# @TEST-EXEC: cat $LOGS/conn.log | zeek-cut -C -d ts id.orig_h ts > ts-twice +# @TEST-EXEC: btest-diff ts-twice + +# File has more than one time column. +# @TEST-EXEC: cat $LOGS/multiple-times.log | zeek-cut -C -d proto ts2 ts1 > multiple-times +# @TEST-EXEC: btest-diff multiple-times diff --git a/auxil/zeek-aux/testing/zeek-cut/time-optional.test b/auxil/zeek-aux/testing/zeek-cut/time-optional.test new file mode 100644 index 0000000000..4aea1b9440 --- /dev/null +++ b/auxil/zeek-aux/testing/zeek-cut/time-optional.test @@ -0,0 +1,10 @@ +# Test zeek-cut with time conversion when a time field is optional and no value +# is set. +# + +# @TEST-EXEC: cat $LOGS/time-optional.log | zeek-cut -u > out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-time-zone btest-diff out + +# File has a non-default unset field string. +# @TEST-EXEC: cat $LOGS/unset-field.log | zeek-cut -u > out2 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-time-zone btest-diff out2 diff --git a/auxil/zeek-aux/zeek-archiver/CHANGES.pre-zeek-aux b/auxil/zeek-aux/zeek-archiver/CHANGES.pre-zeek-aux new file mode 100644 index 0000000000..b97185e282 --- /dev/null +++ b/auxil/zeek-aux/zeek-archiver/CHANGES.pre-zeek-aux @@ -0,0 +1,206 @@ +This file contains the CHANGES history of zeek-archiver from prior to its merge +into this repo. Newer updates end up in the toplevel CHANGES file. + +0.7.0-42 | 2023-12-14 19:57:51 -0800 + + * CI updates (Christian Kreibich, Corelight) + + - Remove Fedora 35-37, add 38 & 39 + - Remove Ubuntu 18 + - Remove openSUSE Leap 15.3 and 15.4 (about to EOL), add 15.5 + - Remove macOS Monterey, add Sonoma + - Add Debian 12 + - Take FreeBSD 14 out of test-only mode, bump 13 to 13.2, drop 12 + +0.7.0-40 | 2023-05-09 12:59:42 +0200 + + * Add support for parsing log_suffix metadata from filenames (Arne Welzel, Corelight) + + This is similar to the approach that we took for ZeekControl, tagging a + log_suffix to the archived file. With zeek-archiver, however, we can't + inject an environment variable that denotes the logger - there's only a + single zeek-archiver instance. Instead, the logger will encode that + information into filename: + + test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-1__.log + + Outside of somehow adding a side-channel, the filename is all we have to + propagate this information. I didn't like the idea of having the position + imply the meaning, so key=value it became. It currently only recognizes + the log_suffix metadata. + +0.7.0-38 | 2023-01-11 16:47:28 -0800 + + * CI: drop macOS Big Sur, add macOS Ventura (Christian Kreibich, Corelight) + + * CI: remove Fedora 35, now EOL (Christian Kreibich, Corelight) + +0.7.0-35 | 2022-11-29 12:09:32 -0800 + + * CI updates (Christian Kreibich, Corelight) + + - remove FreeBSD 11, EOL + - actually run Fedora 36 ... + - add Fedora 37 (Christian Kreibich, Corelight) + + * Add CodeQL workflow for GitHub code scanning (LGTM Migrator) + +0.7.0-29 | 2022-07-07 14:15:07 -0700 + + * CI updates (Christian Kreibich, Corelight) + + - add FreeBSD 13.1 and fix broken link + - add Ubuntu 22.04 + - add OpenSUSE Leap 15.4, OpenSUSE Tumbleweed + - remove Ubuntu 21.10, EOL 2022-07-14 + - remove Debian 9, EOL 2022-06-30 + - drop Fedora 34 (EOL 2022-06-07), add Fedora 36 + + * Make top-level Makefile consistent with other zeek subprojects (Tim Wojtulewicz, Corelight) + +0.7.0-21 | 2022-05-05 12:07:42 -0700 + + * CI updates (Christian Kreibich, Corelight) + + - add FreeBSD 14 + - add Ubuntu 21.10 + - remove OpenSUSE Leap 15.2 (EOL) + - add CentOS Stream 9 + - remove Fedora 33 and CentOS 8 + +0.7.0-15 | 2021-11-05 12:44:52 -0700 + + * Add macOS Monterey and drop Catalina in CI (Christian Kreibich, Corelight) + + * Add Fedora 35 to CI (Christian Kreibich, Corelight) + +0.7.0-11 | 2021-08-26 14:28:39 -0700 + + * CI support refresh (Christian Kreibich, Corelight) + + - Add Debian 11 (Bullseye) + - Drop Ubuntu 16.04 + +0.7.0-9 | 2021-06-21 16:13:02 -0700 + + * CI support refresh to bring in line with Zeek (Christian Kreibich, Corelight) + +0.7.0-7 | 2021-05-14 11:08:45 -0700 + + * Fixed typo in --help output (copeland3300) + +0.7.0-4 | 2021-01-19 21:05:03 -0800 + + * Update Cirrus CI naming for MacOS images (Christian Kreibich, Corelight) + + The docs on https://cirrus-ci.org/guide/macOS/ changed from + "osx_instance" to "macos_instance", so let's reflect that. + +0.7.0-2 | 2020-12-21 10:38:57 -0800 + + * Add macOS Big Sur to CI (Jon Siwek, Corelight) + +0.7.0 | 2020-12-14 20:46:15 -0800 + + * Release 0.7.0 + + * Add 'dist' target to Makefile (Jon Siwek, Corelight) + +0.6.3-7 | 2020-12-02 12:03:49 -0800 (Jon Siwek, Corelight) + + * Update Cirrus CI config/Dockerfiles + + Keeping in sync with Zeek's currently supported platforms + +0.6.3-5 | 2020-12-02 11:07:12 -0800 + + * Update minimum required CMake to 3.5 (Jon Siwek, Corelight) + +0.6.3-3 | 2020-08-25 17:01:32 -0700 + + * Fix closing timestamp in rotation format function example (Jon Siwek, Corelight) + +0.6.3-2 | 2020-08-19 11:35:47 -0700 + + * Add GH Action for CI email notifications (Jon Siwek, Corelight) + + * Add example of how to use with ZeekControl (Jon Siwek, Corelight) + +0.6.3 | 2020-07-28 16:03:04 -0700 + + * Release 0.6.3. + +0.6.2-4 | 2020-07-28 16:00:12 -0700 + + * Add SIGTERM handler to kill/wait any child-compression process (Jon Siwek, Corelight) + + * Prevent command injection through filenames when running compression command (Arne Welzel, Corelight) + + * Allow empty file extension argument in --compress (Jon Siwek, Corelight) + +0.6.2 | 2020-07-27 10:10:41 -0700 + + * Release 0.6.2. + +0.6.1-3 | 2020-07-27 10:09:42 -0700 + + * service: Use a more contained setup (Arne Welzel, Corelight) + + - do not run as root + - don't allow privilege escalation + - only allow read-write to /usr/local/zeek/logs + + * service: Make Description identify the unit instead of describing it (Arne Welzel, Corelight) + + The official docs have this gem: + + This is used by systemd (and other UIs) as the label for the unit, + so this string should identify the unit rather than describe it, + despite the name. + + ...and the logs look a bit nicer afterwards: + + Jul 25 12:25:17 tinkyx280 systemd[1]: Started Zeek Archiver atomically rotates/compresses Zeek logs. + Jul 25 12:26:32 tinkyx280 systemd[1]: Started Zeek Archiver + +0.6.1 | 2020-07-21 12:26:33 -0700 + + * Release 0.6.1. + +0.6.0-2 | 2020-07-21 12:26:11 -0700 + + * Improve documentation (Jon Siwek, Corelight) + +0.6.0-1 | 2020-07-20 15:43:48 -0700 + + * Extend README with implementation rationale (Jon Siwek, Corelight) + +0.6.0 | 2020-07-16 17:09:08 -0700 + + * Release 0.6.0 + + * Add tests (Jon Siwek, Corelight) + +0.5.0-6 | 2020-07-16 15:53:57 -0700 + + * Add CI config (Jon Siwek, Corelight) + +0.5.0-5 | 2020-07-16 13:23:40 -0700 + + * Update README dependency info (Jon Siwek, Corelight) + +0.5.0-4 | 2020-07-16 12:39:54 -0700 + + * Tell CMake to generate compile_commands.json (Jon Siwek, Corelight) + +0.5.0-3 | 2020-07-16 12:30:51 -0700 + + * Fix race condition in make_dir() (Jon Siwek, Corelight) + +0.5.0-2 | 2020-07-16 11:31:20 -0700 + + * Add CHANGES file (Jon Siwek, Corelight) + +0.5.0-1 | 2020-07-16 11:25:17 -0700 + + * Starting CHANGES. diff --git a/auxil/zeek-aux/zeek-archiver/CMakeLists.txt b/auxil/zeek-aux/zeek-archiver/CMakeLists.txt new file mode 100644 index 0000000000..e2523e0b53 --- /dev/null +++ b/auxil/zeek-aux/zeek-archiver/CMakeLists.txt @@ -0,0 +1,5 @@ +add_executable(zeek-archiver zeek-archiver.cc) + +target_compile_options(zeek-archiver PRIVATE -Wall -Wno-unused -Werror=vla) + +install(TARGETS zeek-archiver) diff --git a/auxil/zeek-aux/zeek-archiver/COPYING b/auxil/zeek-aux/zeek-archiver/COPYING new file mode 100644 index 0000000000..35d65d1172 --- /dev/null +++ b/auxil/zeek-aux/zeek-archiver/COPYING @@ -0,0 +1,28 @@ +Copyright (c) 2020-2023 by Corelight, Inc + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +(1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +(2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +(3) Neither the name of Corelight, Inc, nor the names of contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/auxil/zeek-aux/zeek-archiver/README.md b/auxil/zeek-aux/zeek-archiver/README.md new file mode 100644 index 0000000000..8466b402c6 --- /dev/null +++ b/auxil/zeek-aux/zeek-archiver/README.md @@ -0,0 +1,93 @@ +# zeek-archiver + +A Zeek log archival service. + +This tool is derived from +[bro-atomic-rotate](https://github.com/ncsa/bro-atomic-rotate) +and intends to solve two failings of Zeek's historical log-archival +process: robustness and atomicity. It's rewritten with two further +requirements in mind: + +* Independence from [ZeekControl](https://github.com/zeek/zeekctl). + It's meant for use in conjunction with the upcoming + [Zeek Supervisor Framework](https://docs.zeek.org/en/current/frameworks/supervisor.html). +* Independence from Python. In retrospect, it's dubious whether that's + a benefit: implementing in C++ has little advantage over a solution done in + a simpler Bash/Python script, so it may get completely rewritten + later if any concrete maintenance burden/problems are encountered. + +## Dependencies + +* CMake 3.0 or greater +* C++ compiler with C++17 support (GCC 7+ or Clang 4+) +* By default, compression is enabled by shelling out directly to `gzip`, + either install that separately or disable/change the compression + mechanism via the `--compression=` flag. + +## Installation + +Since `zeek-archiver` is made for use with the Zeek Supervisor Framework, +you should first install Zeek and configure your Supervised Cluster, based +on the example given here which will rotate logs into `$(cwd)/logger/log-queue/`: +https://docs.zeek.org/en/current/frameworks/supervisor.html#supervised-cluster-example + +After, install/configure `zeek-archiver` itself as a service: + +``` +$ make install +$ cp zeek-archiver.service /etc/systemd/system/ +# Modify the ExecStart invocation in service file as needed. +$ systemctl enable zeek-archiver +$ systemctl start zeek-archiver +``` + +## Use With ZeekControl + +While `zeek-archiver` is meant to be used with the Zeek Supervisor Framework, +it's still possible to use with ZeekControl in the time before it's entirely +succeeded by the Supervisor Framework. As an example of how to configure +`zeek-archiver` to work with ZeekControl, add this code to your `local.zeek` + +```zeek +@if ( Cluster::local_node_type() == Cluster::LOGGER ) + +redef Log::default_rotation_dir = "/usr/local/zeek/logs/log-queue"; + +function my_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath + { + local open_str = strftime(Log::default_rotation_date_format, ri$open); + local close_str = strftime(Log::default_rotation_date_format, ri$close); + local base = fmt("%s__%s__%s__", ri$path, open_str, close_str); + local rval = Log::RotationPath($file_basename=base); + return rval; + } + +redef Log::rotation_format_func = my_rotation_format_func; +redef Log::default_rotation_postprocessor_cmd = ""; + +@endif +``` + +Then run `zeek-archiver` the same way as explained earlier to have it monitor +and rotate any logs that show up in `/usr/local/zeek/logs/log-queue`. + +## Further Background + +The historical ZeekControl method for log rotation/archival looked like: + +``` +mv conn.log conn-yaddayadda.log +gzip < conn-yaddayadda.log > /bro/logs/2018/10/10/conn.09:00:00-10:00:00.gz +``` + +But that is not an "atomic" operation that's robust in the face of power less, +reboot, OOM, something trying to read `.gz` files as they're created. +The archival process for each log also happened all concurrently with one +another, which creates problematic load spikes. + +Instead, `zeek-archiver` archives log files serially and atomically in a way +that depends on which criteria is met: + +* If compression is desired: `gzip < src > dst.tmp && mv dst.tmp dst && rm src` +* No compression, within same filesystem: `mv src dst` +* No compression, across filesystems: `cp src dst.tmp && mv dst.tmp dst && rm src` diff --git a/auxil/zeek-aux/zeek-archiver/zeek-archiver.cc b/auxil/zeek-aux/zeek-archiver/zeek-archiver.cc new file mode 100644 index 0000000000..be1e0e5e08 --- /dev/null +++ b/auxil/zeek-aux/zeek-archiver/zeek-archiver.cc @@ -0,0 +1,870 @@ +#define _XOPEN_SOURCE +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr auto ZEEK_ARCHIVER_VERSION = "v0.50-174"; + +struct Options { + std::string src_dir; + std::string dst_dir; + + bool verbose = false; + bool oneshot = false; + std::string delimiter = "__"; + std::string compress_ext = "gz"; + std::string compress_cmd = "gzip"; + std::string timestamp_fmt = "%Y-%m-%d-%H-%M-%S"; + std::vector zip_file_extensions = { "gz", "bz2", "lz", "lz4" }; + + int idle_poll_interval = 30; +}; + +static Options options; + +struct LogFile { + std::string path; + std::string name; + struct tm open; + struct tm close; + std::string ext; + std::string suffix; + + std::string DestDir() const + { + char buf[64]; + auto res = strftime(buf, sizeof(buf), "%Y-%m-%d", &open); + + if ( res == 0 ) + return {}; + + return buf; + } + + std::string DestFile() const + { + constexpr auto time_fmt = "%H:%M:%S"; + char buf[64]; + auto res = strftime(buf, sizeof(buf), time_fmt, &open); + + if ( res == 0 ) + return {}; + + std::string start = buf; + + res = strftime(buf, sizeof(buf), time_fmt, &close); + + if ( res == 0 ) + return {}; + + std::string close = buf; + + std::string r = name + "." + start + "-" + close; + if ( ! suffix.empty() ) + r += "-" + suffix; + + return r + ext; + } +}; + +static double now() + { + struct timeval tv; + + if ( gettimeofday(&tv, 0) < 0 ) + return 0; + + return (double)tv.tv_sec + (double)tv.tv_usec / 1e6; + } + +static void debug(const char* format, ...) __attribute__((format (printf, 1, 2))); +static void debug(const char* format, ...) + { + if ( ! options.verbose ) + return; + + auto f = stdout; + fprintf(f, "[%17.06f] [DEBUG] ", now()); + + va_list args; + va_start(args, format); + vfprintf(f, format, args); + va_end(args); + + fprintf(f, "\n"); + } + +static void info(const char* format, ...) __attribute__((format (printf, 1, 2))); +static void info(const char* format, ...) + { + auto f = stdout; + fprintf(f, "[%17.06f] [INFO] ", now()); + + va_list args; + va_start(args, format); + vfprintf(f, format, args); + va_end(args); + + fprintf(f, "\n"); + } + +static void error(const char* format, ...) __attribute__((format (printf, 1, 2))); +static void error(const char* format, ...) + { + auto f = stderr; + fprintf(f, "[%17.06f] [ERROR] ", now()); + + va_list args; + va_start(args, format); + vfprintf(f, format, args); + va_end(args); + + fprintf(f, "\n"); + } + +static void fatal(const char* format, ...) __attribute__((format (printf, 1, 2))); +static void fatal(const char* format, ...) + { + auto f = stderr; + fprintf(f, "[%17.06f] [FATAL] ", now()); + + va_list args; + va_start(args, format); + vfprintf(f, format, args); + va_end(args); + + fprintf(f, "\n"); + exit(1); + } + +static void print_version(FILE* f) + { + fprintf(f, "zeek-archiver %s\n", ZEEK_ARCHIVER_VERSION); + } + +static void print_usage() + { + print_version(stderr); + fprintf(stderr, "usage: zeek-archiver [options] \n"); + fprintf(stderr, " | A directory to monitor for Zeek log files\n"); + fprintf(stderr, " | A directory to archive Zeek logs into\n"); + fprintf(stderr, " --version | Print version and exit\n"); + fprintf(stderr, " -1 | Archive current logs and exit w/o looping\n"); + fprintf(stderr, " -h|--help | Show this usage information\n"); + fprintf(stderr, " -v|--verbose | Print verbose/debug logs to stderr\n"); + fprintf(stderr, " -c|--compress | File extension and compression command,\n" + " empty string means \"disable compression\"\n" + " (default: \"gz,gzip\")\n"); + fprintf(stderr, " -d|--delimiter | Delimiter between timestamps in log names\n" + " (default: \"__\")\n"); + fprintf(stderr, " -t|--time-fmt | Format of timestamps within input file names\n" + " (default: \"%%Y-%%m-%%d-%%H-%%M-%%S\")\n"); + fprintf(stderr, " -z|--zip-extensions | File extensions for already-zipped logs,\n" + " an empty string disables this feature\n" + " (default: \"gz,bz2,lz,lz4\")\n"); + } + +static void usage_error(const char* format, ...) __attribute__((format (printf, 1, 2))); +static void usage_error(const char* format, ...) + { + print_usage(); + + fprintf(stderr, "ERROR: "); + + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + + fprintf(stderr, "\n"); + + exit(1); + } + +static std::vector +split_string(std::string_view input, std::string_view delim) + { + std::vector rval; + size_t pos = 0; + size_t n = 0; + + while ( (n = input.find(delim, pos)) != std::string::npos ) + { + rval.emplace_back(input.substr(pos, n - pos)); + pos = n + delim.size(); + } + + rval.emplace_back(input.substr(pos)); + return rval; + } + +static std::string strip_string(std::string s) + { + auto notspace = [](unsigned char c) + { + return ! std::isspace(c); + }; + s.erase(s.begin(), std::find_if(s.begin(), s.end(), notspace)); + s.erase(std::find_if(s.rbegin(), s.rend(), notspace).base(), s.end()); + return s; + } + +static void consume_option_value(const std::string& flag, std::string arg_value) + { + if ( flag == "-c" || flag == "--compress" ) + { + if ( arg_value.empty() ) + options.compress_cmd = ""; + else + { + auto parts = split_string(arg_value, ","); + + if ( parts.size() != 2 ) + usage_error("--compress must give a 'ext,compress_cmd' formatted " + "value, got: %s", arg_value.data()); + + options.compress_ext = parts[0]; + options.compress_cmd = parts[1]; + } + } + + + else if ( flag == "-d" || flag == "--delimiter" ) + { + if ( arg_value.empty() ) + usage_error("flag '%s' is missing a value", flag.data()); + + options.delimiter = std::move(arg_value); + } + + else if ( flag == "-t" || flag == "--time-fmt" ) + { + if ( arg_value.empty() ) + usage_error("flag '%s' is missing a value", flag.data()); + + options.timestamp_fmt = std::move(arg_value); + } + + else if ( flag == "-z" || flag == "--zip-extensions" ) + { + options.zip_file_extensions = split_string(arg_value, ","); + } + } + +static void parse_options(int argc, char** argv) + { + std::set flags = { + "--version", + "-1", + "-h", "--help", + "-v", "--verbose", + "-c", "--compress", + "-d", "--delimiter", + "-t", "--time-fmt", + "-z", "--zip-extensions", + }; + + bool in_options = true; + + for ( auto i = 1; i < argc; ++i ) + { + auto arg = argv[i]; + + if ( ! arg[0] ) + continue; + + if ( arg[0] == '-' ) + { + if ( ! in_options ) + usage_error("optional flags must precede non-optional arguments: " + "'%s'", arg); + + if ( ! arg[1] ) + // Has to be something after a '-' + usage_error("invalid argument: '%s'", arg); + + if ( arg[1] != '-' && arg[2] && arg[2] != '=' ) + // Invalid short flag: must be -x, -x v, or -x=v + usage_error("invalid argument: '%s'", arg); + + std::string flag = arg; + std::string opt_value; + + auto it = flag.find('='); + + if ( it == std::string::npos ) + { + if ( i + 1 < argc ) + opt_value = argv[i + 1]; + } + else + { + opt_value = flag.substr(it + 1); + flag = flag.substr(0, it); + } + + if ( flags.find(flag) == flags.end() ) + usage_error("invalid argument: '%s'", arg); + + if ( flag == "-1" ) + { + if ( ! opt_value.empty() && it != std::string::npos ) + usage_error("invalid argument=value: '%s'", arg); + + options.oneshot = true; + continue; + } + + if ( flag == "--version" ) + { + if ( ! opt_value.empty() && it != std::string::npos ) + usage_error("invalid argument=value: '%s'", arg); + + print_version(stdout); + exit(0); + } + + if ( flag == "-h" || flag == "--help" ) + { + if ( ! opt_value.empty() && it != std::string::npos ) + usage_error("invalid argument=value: '%s'", arg); + + print_usage(); + exit(0); + } + + if ( flag == "-v" || flag == "--verbose" ) + { + if ( ! opt_value.empty() && it != std::string::npos ) + usage_error("invalid argument=value: '%s'", arg); + + options.verbose = true; + continue; + } + + if ( it == std::string::npos ) + ++i; + + consume_option_value(flag, std::move(opt_value)); + continue; + } + else + { + if ( options.src_dir.empty() ) + { + in_options = false; + options.src_dir = arg; + } + else if ( options.dst_dir.empty() ) + { + in_options = false; + options.dst_dir = arg; + } + else + usage_error("extra/invalid argument: '%s': / " + "already provided: %s/%s", arg, + options.src_dir.data(), options.dst_dir.data()); + } + } + + if ( options.src_dir.empty() ) + usage_error("no provided"); + + if ( options.dst_dir.empty() ) + usage_error("no provided"); + } + +static bool make_dir(const char* dir) + { + if ( mkdir(dir, 0775) == 0 ) + return true; + + auto mkdir_errno = errno; + struct stat st; + + if ( stat(dir, &st) == -1 ) + { + // Show the original failure reason for mkdir() since nothing's there + // or we can't even tell what is now. + error("Failed to create directory %s: %s", dir, strerror(mkdir_errno)); + return false; + } + + if ( S_ISDIR(st.st_mode) ) + return true; + + error("Failed to create directory %s: exists but is not a directory", dir); + return false; + } + +static bool make_dirs(std::string_view dir) + { + auto parts = split_string(dir, "/"); + std::string current_dir = dir[0] == '/' ? "/" : ""; + std::vector dirs; + + for ( auto& p : parts ) + if ( ! p.empty() ) + dirs.emplace_back(std::move(p)); + + for ( size_t i = 0; i < dirs.size(); ++i ) + { + if ( i > 0 ) + current_dir += '/'; + + current_dir += dirs[i]; + + if ( ! make_dir(current_dir.data()) ) + return false; + } + + return true; + } + +bool is_file(const char* path) + { + struct stat st; + + if ( stat(path, &st) == -1 ) + { + if ( errno != ENOENT ) + error("can't stat %s: %s", path, strerror(errno)); + + return false; + } + + return S_ISREG(st.st_mode); + } + +std::optional same_filesystem(const char* path1, const char* path2) + { + struct stat st1; + struct stat st2; + + if ( stat(path1, &st1) == -1 ) + { + error("can't stat %s: %s", path1, strerror(errno)); + return {}; + } + + if ( stat(path2, &st2) == -1 ) + { + error("can't stat %s: %s", path2, strerror(errno)); + return {}; + } + + return st1.st_dev == st2.st_dev; + } + +static bool ends_with(std::string_view s, std::string_view ending) + { + if ( ending.size() > s.size() ) + return false; + + return std::equal(ending.rbegin(), ending.rend(), s.rbegin()); + } + +static bool already_zipped(std::string_view file) + { + for ( const auto& e : options.zip_file_extensions ) + if ( ends_with(file, e) ) + return true; + + return false; + } + +static pid_t child_pid = -1; + +static void signal_handler(int signal) + { + if ( child_pid > 0 ) + { + kill(child_pid, SIGKILL); + int status; + waitpid(child_pid, &status, 0); + } + + _exit(131); + } + +// Fork a child and associate its stdin/stdout with the src and dst files, +// then run compress_cmd via system(). +static int run_compress_cmd(const char* src_file, const char* dst_file) + { + child_pid = fork(); + + if ( child_pid == -1 ) + { + error("Failed to fork() to run compress command: %s", strerror(errno)); + return -1; + } + + if ( child_pid == 0 ) + { + int src_fd = open(src_file, O_RDONLY); + + if ( src_fd < 0 ) + { + error("Failed to open src_file %s: %s", src_file, strerror(errno)); + exit(254); + } + + if ( dup2(src_fd, STDIN_FILENO) == -1 ) + { + error("Failed to redirect src_file %s to stdin: %s", src_file, + strerror(errno)); + exit(253); + } + + if ( src_fd != STDIN_FILENO ) + close(src_fd); + + int dst_fd = open(dst_file, O_CREAT | O_TRUNC | O_WRONLY, 0664); + + if ( dst_fd < 0 ) + { + error("Failed to open dst_file %s: %s", dst_file, strerror(errno)); + exit(252); + } + + if ( dup2(dst_fd, STDOUT_FILENO) == -1 ) + { + error("Failed to redirect dst_file %s to stdout: %s", dst_file, + strerror(errno)); + exit(251); + } + + if ( dst_fd != STDOUT_FILENO ) + close(dst_fd); + + // Call the compression program via the shell. + execlp("sh", "sh", "-c", options.compress_cmd.data(), (char*)0); + error("Failed to exec(): %s", strerror(errno)); + exit(255); + } + + int status; + waitpid(child_pid, &status, 0); + child_pid = -1; + + if ( ! (WIFEXITED(status) && WEXITSTATUS(status) == 0) ) + { + if ( WIFEXITED(status) ) + error("Compression of %s failed, command exit status: %d (0x%x)", + src_file, WEXITSTATUS(status), status); + else if ( WIFSIGNALED(status) ) + error("Compression of %s failed, got signal: %d (0x%x)", + src_file, WTERMSIG(status), status); + else + error("Compression of %s failed, unknown reason/status: (0x%x)", + src_file, status); + + // If the compression command failed, unlink the destination + // file. Ignore any errors - it may not have been created. + unlink(dst_file); + } + + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; + } + +static int archive_logs() + { + int rval = 0; + + auto d = opendir(options.src_dir.data()); + + if ( ! d ) + { + debug("Source directory '%s', does not exist", options.src_dir.data()); + return rval; + } + + struct dirent* dp; + std::vector log_files; + + while ( (dp = readdir(d)) ) + { + if ( dp->d_name[0] == '.' ) + continue; + + std::string path = options.src_dir + "/" + dp->d_name; + + if ( ! is_file(path.data()) ) + { + debug("Skipping archival of non-file: %s", dp->d_name); + continue; + } + + // Default log file format either has 4 parts delimited by "__", + // as follows: + // + // test__2020-07-16-09-43-10__2020-07-16-09-43-10__.log + // + // Or, 5 parts delimited by "__" where the part before the extension + // is a generic comma separated key=value construct: + // + // test__2020-07-16-09-43-10__2020-07-16-09-43-10__log_suffix=logger-1,pid=4711__.log + // + // The comma character is reasonable to work with on a shell and assumed + // to not be of importance for metadata values. If this seems over-engineered, + // maybe, but adding a plain positional parameter with an implied meaning also + // adds a required parameter for any future extensions and we currently don't + // have a side-channel to propagate additional information. + // + auto parts = split_string(dp->d_name, options.delimiter); + + if ( parts.size() != 4 && parts.size() != 5) + { + debug("Skipping archival of non-log: %s", dp->d_name); + continue; + } + + LogFile lf; + lf.path = path; + lf.name = parts[0]; + + auto res = strptime(parts[1].data(), options.timestamp_fmt.data(), &lf.open); + + if ( ! res ) + { + debug("Skipping archival of log with bad timestamp format: %s", dp->d_name); + continue; + } + + if ( res != parts[1].data() + parts[1].size() ) + debug("Possible log with timestamp format mismatch: %s", dp->d_name); + + res = strptime(parts[2].data(), options.timestamp_fmt.data(), &lf.close); + + if ( ! res ) + { + debug("Skipping archival of log with bad timestamp format: %s", dp->d_name); + continue; + } + + if ( res != parts[2].data() + parts[2].size() ) + debug("Possible log with timestamp format mismatch: %s", dp->d_name); + + if ( parts.size() == 4 ) + lf.ext = parts[3]; + else + { + lf.ext = parts[4]; + + bool metadata_error = false; + + // split_string() returns a single entry for + // an empty string, avoid that scenario. + std::vector metadata_parts; + if ( ! parts[3].empty() ) + metadata_parts = split_string(parts[3], ","); + + for (const auto& entry : metadata_parts) + { + auto key_value = split_string(entry, "="); + if ( key_value.size() != 2 ) + { + metadata_error = true; + break; + } + + auto key = strip_string(key_value[0]); + auto value = strip_string(key_value[1]); + if ( key.empty() || value.empty() ) + { + metadata_error = true; + break; + } + + // Only log_suffix is understood as metadata. + if ( key == "log_suffix" ) + { + debug("Using log_suffix '%s'", value.data()); + lf.suffix = value; + } + else + debug("Ignoring unknown metadata entry %s in %s", key.data(), dp->d_name); + + } + + if ( metadata_error ) + { + debug("Skipping archival of log with bad metadata format: %s", dp->d_name); + continue; + } + } + + log_files.emplace_back(std::move(lf)); + } + + closedir(d); + + for ( const auto& lf : log_files ) + { + auto dst_dir = options.dst_dir + "/" + lf.DestDir(); + auto dst_file = dst_dir + "/" + lf.DestFile(); + auto tmp_file = dst_dir + "/.tmp." + lf.DestFile(); + const auto& src_file = lf.path; + + if ( ! make_dirs(dst_dir) ) + { + error("Skipped archiving %s: failed to create dir %s", + src_file.data(), dst_dir.data()); + continue; + } + + bool compress = ! options.compress_cmd.empty() && ! already_zipped(lf.ext); + + if ( compress ) + { + if ( ! options.compress_ext.empty() ) + dst_file += "." + options.compress_ext; + + debug("Archive via compression: %s -> %s", src_file.data(), dst_file.data()); + auto res = run_compress_cmd(src_file.data(), tmp_file.data()); + + if ( res != 0 ) + continue; + + res = rename(tmp_file.data(), dst_file.data()); + + if ( res == -1 ) + { + error("Failed to rename %s -> %s: %s", + tmp_file.data(), dst_file.data(), strerror(errno)); + continue; + } + + ++rval; + res = unlink(src_file.data()); + + if ( res == -1 ) + error("Failed to unlink %s; %s", src_file.data(), strerror(errno)); + + continue; + } + + auto same_fs = same_filesystem(src_file.data(), dst_dir.data()); + + if ( ! same_fs ) + { + error("Failed to compare filesystems of %s and %s", + src_file.data(), dst_dir.data()); + continue; + } + + if ( *same_fs ) + { + debug("Archive via rename: %s -> %s", src_file.data(), dst_file.data()); + auto res = rename(src_file.data(), dst_file.data()); + + if ( res == -1 ) + error("Failed to rename %s -> %s: %s", + src_file.data(), dst_file.data(), strerror(errno)); + else + ++rval; + } + else + { + debug("Archive via copy: %s -> %s", src_file.data(), dst_file.data()); + + std::ifstream src; + std::ofstream dst; + src.exceptions(std::ifstream::failbit | std::ifstream::badbit); + dst.exceptions(std::ofstream::failbit | std::ofstream::badbit); + + try + { + src.open(src_file, std::ios::binary); + dst.open(tmp_file, std::ios::binary); + dst << src.rdbuf(); + } + catch ( const std::system_error& e ) + { + error("Failed to copy %s to temporary file %s: %s", + src_file.data(), tmp_file.data(), e.code().message().data()); + continue; + } + + auto res = rename(tmp_file.data(), dst_file.data()); + + if ( res == -1 ) + { + error("Failed to rename %s -> %s: %s", + tmp_file.data(), dst_file.data(), strerror(errno)); + continue; + } + + ++rval; + res = unlink(src_file.data()); + + if ( res == -1 ) + error("Failed to unlink %s; %s", src_file.data(), strerror(errno)); + } + } + + return rval; + } + +int main(int argc, char** argv) + { + signal(SIGTERM, signal_handler); + parse_options(argc, argv); + + debug("Using src_dir: '%s'", options.src_dir.data()); + debug("Using dst_dir: '%s'", options.dst_dir.data()); + debug("Using oneshot option: '%d'", options.oneshot); + debug("Using delimiter option: '%s'", options.delimiter.data()); + debug("Using timestamp format option: '%s'", options.timestamp_fmt.data()); + debug("Using compression extension option: '%s'", options.compress_ext.data()); + debug("Using compression command option: '%s'", options.compress_cmd.data()); + debug("Using poll interval: '%d'", options.idle_poll_interval); + + for ( const auto& e : options.zip_file_extensions ) + debug("Using zip-extension option: '%s'", e.data()); + + if ( ! make_dirs(options.dst_dir) ) + fatal("Failed to create destination archive dir: %s", options.dst_dir.data()); + + for ( ; ; ) + { + using hrc = std::chrono::high_resolution_clock; + auto t0 = hrc::now(); + auto num_archived = archive_logs(); + auto t1 = hrc::now(); + + if ( num_archived > 0 ) + { + auto dt = std::chrono::duration(t1 - t0).count(); + info("Archived %d logs in %f seconds", num_archived, dt); + } + + if ( options.oneshot ) + break; + + sleep(num_archived > 0 ? 1 : options.idle_poll_interval); + } + + return 0; + } diff --git a/auxil/zeek-aux/zeek-archiver/zeek-archiver.service b/auxil/zeek-aux/zeek-archiver/zeek-archiver.service new file mode 100644 index 0000000000..2440e267e8 --- /dev/null +++ b/auxil/zeek-aux/zeek-archiver/zeek-archiver.service @@ -0,0 +1,21 @@ +[Unit] +Description=Zeek Archiver +After=network.target + +[Service] +# Run as unprivileged user, never allow privilege escalation and mount +# everything read-only except for where the source and destination +# directories reside. +User=zeek +Group=zeek +NoNewPrivileges=yes +ProtectSystem=strict +ReadWritePaths=/usr/local/zeek/logs + +Nice=10 +ExecStart=/usr/local/zeek/bin/zeek-archiver -v /usr/local/zeek/logs/log-queue /usr/local/zeek/logs +Restart=on-failure +RestartSec=10s + +[Install] +WantedBy=multi-user.target diff --git a/auxil/zeek-aux/zeek-cut/CMakeLists.txt b/auxil/zeek-aux/zeek-cut/CMakeLists.txt new file mode 100644 index 0000000000..d1231f58d6 --- /dev/null +++ b/auxil/zeek-aux/zeek-cut/CMakeLists.txt @@ -0,0 +1,6 @@ +set(zeekcut_SRCS zeek-cut.c) + +add_executable(zeek-cut ${zeekcut_SRCS}) + +install(TARGETS zeek-cut) +install(FILES zeek-cut.1 DESTINATION ${ZEEK_MAN_INSTALL_PATH}/man1) diff --git a/auxil/zeek-aux/zeek-cut/zeek-cut.1 b/auxil/zeek-aux/zeek-cut/zeek-cut.1 new file mode 100644 index 0000000000..27a88ddd3b --- /dev/null +++ b/auxil/zeek-aux/zeek-cut/zeek-cut.1 @@ -0,0 +1,75 @@ +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. +.TH ZEEK-CUT "1" "November 2014" "zeek-cut " "User Commands" +.SH NAME +zeek-cut \- parse Zeek logs +.SH SYNOPSIS +.B zeek\-cut +[\fIoptions\fR] [\fIcolumns\fR] +.SH DESCRIPTION +Extracts the given columns from ASCII Zeek logs on standard input, and outputs +them to standard output. If no field names are given, all are selected. By +default, +.B zeek\-cut +does not include format header blocks in the output. +.PP +Columns are specified as a list of space-separated field names. The order of +field names given to \fBzeek-cut\fR determines the output order, +which means \fBzeek-cut\fR can be used to reorder columns. +.PP +The ASCII Zeek logs read on standard input must have intact format header +blocks because \fBzeek-cut\fR needs this information to correctly interpret the +log file format. In fact, \fBzeek-cut\fR can process the concatenation of +multiple ASCII log files that have different column layouts. +.SH OPTIONS +.TP +\fB\-c\fR +Include the first format header block in the output. +.TP +\fB\-C\fR +Include all format header blocks in the output. +.TP +\fB-m\fR +Include the first format header block in the output in minimal view. +.TP +\fB-M\fR +Include all format header blocks in the output in minimal view. +.TP +\fB\-d\fR +Convert time values into human\-readable format. +.HP +\fB\-D\fR Like \fB\-d\fR, but specify format for time (see strftime(3) for syntax). +.HP +\fB\-F\fR Sets a different output field separator character. +.TP +\fB\-h\fR +Show help. +.TP +\fB\-n\fR +Print all fields except those specified. +.TP +\fB\-u\fR +Like \fB\-d\fR, but print timestamps in UTC instead of local time. +.HP +\fB\-U\fR Like \fB\-D\fR, but print timestamps in UTC instead of local time. +.SH ENVIRONMENT +.TP +.B ZEEK_CUT_TIMEFMT +For time conversion option \fB\-d\fR or \fB\-u\fR, the format string can be +specified by setting this environment variable. +.SH EXAMPLES +Output three columns and convert time values: +.br +cat conn.log | zeek-cut -d ts id.orig_h id.orig_p +.PP +Output all columns and convert time values with a custom format string: +.br +cat conn.log | zeek-cut -D "%Y-%m-%d %H:%M:%S" +.PP +Compressed logs must be uncompressed with another utility: +.br +zcat conn.log.gz | zeek-cut +.SH SEE ALSO +strftime(3) +.SH AUTHOR +.B zeek-cut +was written by The Zeek Project . diff --git a/auxil/zeek-aux/zeek-cut/zeek-cut.c b/auxil/zeek-aux/zeek-cut/zeek-cut.c new file mode 100644 index 0000000000..9745f17029 --- /dev/null +++ b/auxil/zeek-aux/zeek-cut/zeek-cut.c @@ -0,0 +1,564 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include +#include +// define required for FreeBSD +#define _WITH_GETLINE +#include +#include +#include +#include + +/* The maximum length of converted timestamp that zeek-cut can handle. */ +#define MAX_TIMESTAMP_LEN 100 + +/* User-specified options that stay constant during a run of zeek-cut. */ +struct useropts { + int showhdr; /* show log headers? (0=no, 1=only first, 2=all) */ + int minimalview; /* show headers in minimal view? (0=no, 1=yes) */ + int negate; /* show all but the specified columns? (0=no, 1=yes) */ + int timeconv; /* do time conversion? (0=no, 1=local, 2=UTC) */ + char **columns; /* array of user-specified column names */ + int num_columns; /* number of user-specified column names */ + const char *ofs; /* user-specified output field separator character */ + const char *timefmt; /* strftime format string for time conversion */ +}; + +/* Parameters that might change with each log file being processed. */ +struct logparams { + int *out_indexes; /* array of log file column indices to output */ + int num_out_indexes; /* number of elements in "out_indexes" */ + int idx_range; /* max. value in "out_indexes" plus one */ + int *time_cols; /* array of columns (0=not timestamp, 1=timestamp) */ + char **tmp_fields; /* array of pointers to each field on a line */ + int num_fields; /* number of fields in log file */ + char ifs[2]; /* input field separator character */ + char ofs[2]; /* output field separator character */ + char *unsetf; /* unset field string */ + long prev_ts; /* previous timestamp */ +}; + + +int usage(void) { + puts("\nzeek-cut [options] []\n"); + puts("Extracts the given columns from ASCII Zeek logs on standard input, and outputs"); + puts("them to standard output. If no columns are given, all are selected."); + puts("By default, zeek-cut does not include format header blocks in the output."); + puts("\nExample: cat conn.log | zeek-cut -d ts id.orig_h id.orig_p"); + puts("\n -c Include the first format header block in the output."); + puts(" -C Include all format header blocks in the output."); + puts(" -m Include the first format header blocks in the output in minimal view."); + puts(" -M Include all format header blocks in the output in minimal view."); + puts(" -d Convert time values into human-readable format."); + puts(" -D Like -d, but specify format for time (see strftime(3) for syntax)."); + puts(" -F Sets a different output field separator character."); + puts(" -h Show help."); + puts(" -n Print all fields *except* those specified."); + puts(" -u Like -d, but print timestamps in UTC instead of local time."); + puts(" -U Like -D, but print timestamps in UTC instead of local time.\n"); + puts("For time conversion option -d or -u, the format string can be specified by"); + puts("setting an environment variable ZEEK_CUT_TIMEFMT.\n"); + exit(1); +} + +/* Return the index in "haystack" where "needle" is located (or -1 if not + * found). + */ +int string_index(char *haystack[], int haystack_size, const char *needle) { + int i; + for (i = 0; i < haystack_size; ++i) { + if (!strcmp(haystack[i], needle)) { + return i; + } + } + return -1; +} + +/* Return the input field separator from the log's "#separator " header line. */ +char parsesep(const char *sepstr) { + char ifs; + + if (!strncmp(sepstr, "\\x", 2)) { + long sepval = strtol(sepstr + 2, NULL, 16); + ifs = sepval; + } else { + ifs = sepstr[0]; + } + + return ifs; +} + +/* Determine the columns (if any) where the field is "time". Return 0 for + * success, and non-zero otherwise. + */ +int find_timecol(const char *line, struct logparams *lp) { + int i; + int *tmpptr; + char *copy_of_line; + char *field_ptr; + char *field; + + tmpptr = (int *) realloc(lp->time_cols, lp->idx_range * sizeof(int)); + if (tmpptr == NULL) { + fputs("zeek-cut: out of memory\n", stderr); + return 1; + } + + lp->time_cols = tmpptr; + + if ((copy_of_line = strdup(line)) == NULL) { + fputs("zeek-cut: out of memory\n", stderr); + return 1; + } + field_ptr = copy_of_line; + + int ret = 0; + for (i = 0; i < lp->idx_range; ++i) { + if ((field = strsep(&field_ptr, lp->ifs)) == NULL) { + fputs("zeek-cut: log header does not have enough fields\n", stderr); + ret = 1; + break; + } + + /* Set value of 1 for each "time" column, or 0 otherwise */ + lp->time_cols[i] = strcmp("time", field) ? 0 : 1; + } + + free(copy_of_line); + return ret; +} + +/* Allocate memory for "out_indexes" and store index numbers there + * corresponding to the columns in "line" that we want to output later. + * Set the number of elements in "out_indexes". Also + * store in "idx_range" the maximum value contained in "out_indexes" plus one. + * Return 0 for success, and non-zero otherwise. + */ +int find_output_indexes(char *line, struct logparams *lp, struct useropts *bopts) { + int idx; + int *out_indexes; + char *field_ptr; + char *copy_of_line = NULL; + char *field; + + /* Get the number of fields */ + lp->num_fields = 0; + field = line; + while ((field = strchr(field, lp->ifs[0])) != NULL) { + lp->num_fields++; + field++; + } + lp->num_fields++; + + char **tmpptr; + /* note: size is num_fields+1 because header lines have an extra field */ + tmpptr = (char **) realloc(lp->tmp_fields, (lp->num_fields + 1) * sizeof(char *)); + if (tmpptr == NULL) { + return 1; + } + lp->tmp_fields = tmpptr; + + if (bopts->num_columns == 0) { + /* No columns specified on cmd-line, so use all the columns */ + out_indexes = (int *) realloc(lp->out_indexes, lp->num_fields * sizeof(int)); + if (out_indexes == NULL) { + return 1; + } + + for (idx = 0; idx < lp->num_fields; ++idx) { + out_indexes[idx] = idx; + } + + lp->out_indexes = out_indexes; + lp->idx_range = lp->num_fields; + lp->num_out_indexes = lp->num_fields; + return 0; + } + + /* Set tmp_fields to point to each field on the line */ + if ((copy_of_line = strdup(line)) == NULL) { + return 1; + } + field_ptr = copy_of_line; + + idx = 0; + while ((field = strsep(&field_ptr, lp->ifs)) != NULL) { + lp->tmp_fields[idx++] = field; + } + + int out_idx = 0; + int maxval = 0; + + if (!bopts->negate) { + /* One or more column names were specified on cmd-line */ + out_indexes = (int *) realloc(lp->out_indexes, bopts->num_columns * sizeof(int)); + if (out_indexes == NULL) { + return 1; + } + + for (idx = 0; idx < bopts->num_columns; ++idx) { + out_indexes[idx] = string_index(lp->tmp_fields, lp->num_fields, bopts->columns[idx]); + if (out_indexes[idx] > maxval) { + maxval = out_indexes[idx]; + } + } + out_idx = bopts->num_columns; + } else { + /* The "-n" option was specified on cmd-line */ + out_indexes = (int *) realloc(lp->out_indexes, lp->num_fields * sizeof(int)); + if (out_indexes == NULL) { + return 1; + } + + for (idx = 0; idx < lp->num_fields; ++idx) { + if (string_index(bopts->columns, bopts->num_columns, lp->tmp_fields[idx]) == -1) { + out_indexes[out_idx++] = idx; + if (idx > maxval) { + maxval = idx; + } + } + } + } + + free(copy_of_line); + + lp->out_indexes = out_indexes; + lp->idx_range = maxval + 1; + lp->num_out_indexes = out_idx; + return 0; +} + +/* + * Try to convert a time value to a human-readable timestamp, and then output + * the result. A valid time value is one or more digits followed by a decimal + * point (everything after the decimal point is ignored). If the time + * conversion fails for any reason, then just output the field unmodified. + */ +void output_time(const char *field, struct logparams *lp, struct useropts *bopts) { + /* Buffer is declared static in order to reuse the timestamp string */ + static char tbuf[MAX_TIMESTAMP_LEN]; + + char *tmp; + long tl = strtol(field, &tmp, 10); + + if (tl < 0 || tl == LONG_MAX) { + fprintf(stderr, "zeek-cut: time value out-of-range: %s\n", field); + } else if (*tmp != '.') { + if (strcmp(field, lp->unsetf)) { + /* field is not a valid value and is not the unset field string */ + fprintf(stderr, "zeek-cut: time field is not valid: %s\n", field); + } + } else if (tl == lp->prev_ts) { + /* timestamp is same as the previous one, so skip the conversion */ + fputs(tbuf, stdout); + return; + } else { + time_t tt = tl; + struct tm tmval; + struct tm* tmptr; + tmptr = bopts->timeconv == 1 ? localtime_r(&tt, &tmval) : gmtime_r(&tt, &tmval); + + if (tmptr) { + if (strftime(tbuf, sizeof(tbuf), bopts->timefmt, tmptr)) { + /* output the formatted timestamp */ + fputs(tbuf, stdout); + lp->prev_ts = tl; + return; + } else { + fputs("zeek-cut: failed to convert timestamp (try a shorter format string)\n", stderr); + } + } else { + /* the time conversion will fail for large values */ + fprintf(stderr, "zeek-cut: time value out-of-range: %s\n", field); + } + } + + /* failed to convert, so just output the field without modification */ + fputs(field, stdout); +} + +/* Output the columns of "line" that the user specified. The value of "hdr" + * indicates whether "line" is a header line or not (0=not header, 1=header). + */ +void output_indexes(int hdr, char *line, struct logparams *lp, struct useropts *bopts) { + int i; + char *field; + int dotimeconv = 0; /* do a time conversion on this line? (0=no, 1=yes) */ + int dotimetypeconv = 0; /* change time type on this line? (0=no, 1=yes) */ + int idxrange = lp->idx_range + hdr; /* header lines have one extra field */ + int firstdone = 0; + + /* If user selected time conversion and this line is not a header line, + * then try to do a time conversion. + */ + if (bopts->timeconv && !hdr) { + dotimeconv = 1; + } + + for (i = 0; i < idxrange; ++i) { + if ((field = strsep(&line, lp->ifs)) == NULL) { + fputs("zeek-cut: skipping log line (not enough fields)\n", stderr); + return; + } + lp->tmp_fields[i] = field; + } + + /* If user selected time conversion and this line is a "#types" header, + * then try to change the "time" type field. + */ + if (bopts->timeconv && hdr && !strcmp(lp->tmp_fields[0], "#types")) { + dotimetypeconv = 1; + } + + if (hdr && bopts->minimalview == 0) { + /* Output the initial "#" field on the header line */ + fputs(lp->tmp_fields[0], stdout); + firstdone = 1; + } + + for (i = 0; i < lp->num_out_indexes; ++i) { + int idxval = lp->out_indexes[i]; + + if (firstdone) + fputs(lp->ofs, stdout); + + if (idxval != -1) { + if (dotimeconv && lp->time_cols[idxval]) { + /* output time field */ + output_time(lp->tmp_fields[idxval], lp, bopts); + } else if (dotimetypeconv && !strcmp("time", lp->tmp_fields[idxval + hdr])) { + /* change the "time" type field to "string" */ + fputs("string", stdout); + } else { + /* output the field without modification */ + fputs(lp->tmp_fields[idxval + hdr], stdout); + } + + } + + /* Note: even when idxval == -1, we still need to set "firstdone" so + * that a separator is output. + */ + firstdone = 1; + } + putchar('\n'); +} + +/* Reads one or more log files from stdin and outputs them to stdout according + * to the options specified in "bopts". Returns 0 on success, and non-zero + * otherwise. + */ +int zeek_cut(struct useropts bopts) { + int ret = 0; + struct logparams lp; /* parameters specific to each log file */ + int headers_seen = 0; /* 0=no header blocks seen, 1=one seen, 2=2+ seen */ + int prev_line_hdr = 0; /* previous line was a header line? 0=no, 1=yes */ + int prev_fields_line = 0; /* previous line was #fields line? 0=no, 1=yes */ + ssize_t linelen; + size_t linesize = 100000; + char *line = (char *) malloc(linesize); + + if (line == NULL) { + fputs("zeek-cut: out of memory\n", stderr); + return 1; + } + + lp.out_indexes = NULL; + lp.num_out_indexes = 0; + lp.idx_range = 0; + lp.time_cols = NULL; + lp.tmp_fields = NULL; + lp.num_fields = 0; + lp.ofs[0] = '\t'; + lp.ofs[1] = '\0'; + lp.ifs[0] = '\t'; + lp.ifs[1] = '\0'; + lp.unsetf = strdup("-"); + lp.prev_ts = -1; /* initialize with an invalid time value */ + + if (lp.unsetf == NULL) { + fputs("zeek-cut: out of memory\n", stderr); + free(line); + return 1; + } + + while ((linelen = getline(&line, &linesize, stdin)) > 0) { + /* Remove trailing '\n' */ + line[linelen - 1] = '\0'; + + if (prev_fields_line && strncmp(line, "#types", 6)) { + fputs("zeek-cut: bad log header (missing #types line)\n", stderr); + ret = 1; + break; + } + + /* Check if this line is a header line or not */ + if (line[0] != '#') { + prev_line_hdr = 0; + output_indexes(0, line, &lp, &bopts); + continue; + } + + /* The rest of this loop is for header processing */ + + if (!prev_line_hdr) { + /* Here we are transitioning from non-header to header line */ + prev_line_hdr = 1; + /* Once we've seen two header blocks, we stop counting them */ + if (headers_seen < 2) { + headers_seen++; + } + } + + if (!strncmp(line, "#separator ", 11)) { + char ifs = parsesep(line + 11); + if (ifs == '\0') { + fputs("zeek-cut: bad log header (invalid #separator line)\n", stderr); + ret = 1; + break; + } + + lp.ifs[0] = ifs; + + /* If user-specified ofs is set, then use it. Otherwise, just + * use the log file's input field separator. + */ + lp.ofs[0] = bopts.ofs[0] ? bopts.ofs[0] : lp.ifs[0]; + } else if (!strncmp(line, "#unset_field", 12)) { + if (line[12] && line[13]) { + free(lp.unsetf); + if ((lp.unsetf = strdup(line + 13)) == NULL) { + fputs("zeek-cut: out of memory\n", stderr); + ret = 1; + break; + } + } else { + fputs("zeek-cut: bad log header (invalid #unset_field line)\n", stderr); + ret = 1; + break; + } + } else if (!strncmp(line, "#fields", 7)) { + prev_fields_line = 1; + if (find_output_indexes(line + 8, &lp, &bopts)) { + fputs("zeek-cut: out of memory\n", stderr); + ret = 1; + break; + } + } else if (!strncmp(line, "#types", 6)) { + if (!prev_fields_line) { + fputs("zeek-cut: bad log header (missing #fields line)\n", stderr); + ret = 1; + break; + } + prev_fields_line = 0; + + if (bopts.timeconv) { + if (find_timecol(line + 7, &lp)) { + ret = 1; + break; + } + } + } + + /* Decide if we want to output this header */ + if (bopts.showhdr >= headers_seen) { + if (!strncmp(line, "#fields", 7) || (!strncmp(line, "#types", 6) && (bopts.minimalview == 0))) { + /* Output a modified "#fields" or "#types" header line */ + output_indexes(1, line, &lp, &bopts); + } else if (bopts.minimalview == 0) { + /* Output the header line with no changes */ + puts(line); + } + } + + } + + free(lp.time_cols); + free(lp.out_indexes); + free(lp.tmp_fields); + free(lp.unsetf); + free(line); + return ret; +} + +int main(int argc, char *argv[]) { + int c; + char *envtimefmt = getenv("ZEEK_CUT_TIMEFMT"); + if (envtimefmt == NULL) { + envtimefmt = getenv("BRO_CUT_TIMEFMT"); + if (envtimefmt != NULL) + fprintf(stderr, "zeek-cut warning: using legacy environment variable BRO_CUT_TIMEFMT, set ZEEK_CUT_TIMEFMT instead\n"); + } + + struct useropts bopts; + bopts.showhdr = 0; + bopts.minimalview = 0; + bopts.negate = 0; + bopts.timeconv = 0; + bopts.ofs = ""; + bopts.timefmt = envtimefmt ? envtimefmt : "%Y-%m-%dT%H:%M:%S%z"; + + static struct option long_opts[] = { + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + while ((c = getopt_long(argc, argv, "cCmMnF:duD:U:h", long_opts, NULL)) != -1) { + switch (c) { + case 'c': + bopts.minimalview = 0; + bopts.showhdr = 1; + break; + case 'C': + bopts.minimalview = 0; + bopts.showhdr = 2; + break; + case 'm': + bopts.minimalview = 1; + bopts.showhdr = 1; + break; + case 'M': + bopts.minimalview = 1; + bopts.showhdr = 2; + break; + case 'n': + bopts.negate = 1; + break; + case 'F': + if (strlen(optarg) != 1) { + fputs("zeek-cut: field separator must be a single character\n", stderr); + exit(1); + } + bopts.ofs = optarg; + break; + case 'd': + bopts.timeconv = 1; + break; + case 'u': + bopts.timeconv = 2; + break; + case 'D': + bopts.timeconv = 1; + bopts.timefmt = optarg; + break; + case 'U': + bopts.timeconv = 2; + bopts.timefmt = optarg; + break; + default: + usage(); + break; + } + } + + if (bopts.timeconv && strlen(bopts.timefmt) == 0) { + fputs("zeek-cut: time format string cannot be empty\n", stderr); + exit(1); + } + + bopts.columns = &argv[optind]; + bopts.num_columns = argc - optind; + + return zeek_cut(bopts); +}