diff --git a/CMakeLists.txt b/CMakeLists.txt index d27fa2d40b..28b702ab01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,21 @@ if (GOOGLEPERFTOOLS_FOUND) endif () endif () +set(USE_DATASERIES false) +find_package(Lintel) +find_package(DataSeries) +find_package(LibXML2) + +if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND) + set(USE_DATASERIES true) + include_directories(BEFORE ${Lintel_INCLUDE_DIR}) + include_directories(BEFORE ${DataSeries_INCLUDE_DIR}) + include_directories(BEFORE ${LibXML2_INCLUDE_DIR}) + list(APPEND OPTLIBS ${Lintel_LIBRARIES}) + list(APPEND OPTLIBS ${DataSeries_LIBRARIES}) + list(APPEND OPTLIBS ${LibXML2_LIBRARIES}) +endif() + if (ENABLE_PERFTOOLS_DEBUG) # Just a no op to prevent CMake from complaining about manually-specified # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found @@ -198,6 +213,7 @@ message( "\nGeoIP: ${USE_GEOIP}" "\nGoogle perftools: ${USE_PERFTOOLS}" "\n debugging: ${USE_PERFTOOLS_DEBUG}" + "\nDataSeries: ${USE_DATASERIES}" "\n" "\n================================================================\n" ) diff --git a/NEWS b/NEWS index c48b243552..fe2d9b452e 100644 --- a/NEWS +++ b/NEWS @@ -47,6 +47,13 @@ Bro 2.1 joint set of events. The `icmp_conn` record got a new boolean field 'v6' that indicates whether the ICMP message is v4 or v6. +- Log postprocessor scripts get an additional argument indicating the + type of the log writer in use (e.g., "ascii"). + +- BroControl's make-archive-name scripts also receives the writer + type, but as it's 2nd(!) argument. If you're using a custom version + of that script, you need to adapt it. See the shipped version for + details. TODO: Extend. diff --git a/aux/broctl b/aux/broctl index 519d2e21ee..ed933502b4 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 519d2e21ee375833c89eb6f7dc95c1eac3de17ab +Subproject commit ed933502b4d2518f94b6cfa7a5b371e53fda5c3d diff --git a/aux/btest b/aux/btest index 76876ce0e7..e0da8d0e28 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 76876ce0e7da4888c91b3aea024c5cfd36405310 +Subproject commit e0da8d0e284bbebbaef711c91c1b961580f225d2 diff --git a/cmake b/cmake index 49278736c1..96f3d92aca 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 49278736c1404cb8c077272b80312c947e68bf52 +Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b diff --git a/config.h.in b/config.h.in index b8e2cb9a88..c2cb3ec1dc 100644 --- a/config.h.in +++ b/config.h.in @@ -114,6 +114,9 @@ /* Analyze Mobile IPv6 traffic */ #cmakedefine ENABLE_MOBILE_IPV6 +/* Use the DataSeries writer. */ +#cmakedefine USE_DATASERIES + /* Version number of package */ #define VERSION "@VERSION@" diff --git a/configure b/configure index 85b6af2d7d..3258d4abfc 100755 --- a/configure +++ b/configure @@ -56,6 +56,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-ruby-lib=PATH path to ruby library --with-ruby-inc=PATH path to ruby headers --with-swig=PATH path to SWIG executable + --with-dataseries=PATH path to DataSeries and Lintel libraries + --with-xml2=PATH path to libxml2 installation (for DataSeries) Packaging Options (for developers): --binary-package toggle special logic for binary packaging @@ -213,6 +215,13 @@ while [ $# -ne 0 ]; do --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; + --with-dataseries=*) + append_cache_entry DataSeries_ROOT_DIR PATH $optarg + append_cache_entry Lintel_ROOT_DIR PATH $optarg + ;; + --with-xml2=*) + append_cache_entry LibXML2_ROOT_DIR PATH $optarg + ;; --binary-package) append_cache_entry BINARY_PACKAGING_MODE BOOL true ;; diff --git a/doc/logging-dataseries.rst b/doc/logging-dataseries.rst new file mode 100644 index 0000000000..b41b9fb0b7 --- /dev/null +++ b/doc/logging-dataseries.rst @@ -0,0 +1,168 @@ + +============================= +Binary Output with DataSeries +============================= + +.. rst-class:: opening + + Bro's default ASCII log format is not exactly the most efficient + way for storing large volumes of data. An an alternative, Bro comes + with experimental support for `DataSeries + `_ + output, an efficient binary format for recording structured bulk + data. DataSeries is developed and maintained at HP Labs. + +.. contents:: + +Installing DataSeries +--------------------- + +To use DataSeries, its libraries must be available at compile-time, +along with the supporting *Lintel* package. Generally, both are +distributed on `HP Labs' web site +`_. Currently, however, you need +to use recent developments versions for both packages, which you can +download from github like this:: + + git clone http://github.com/dataseries/Lintel + git clone http://github.com/dataseries/DataSeries + +To build and install the two into ````, do:: + + ( cd Lintel && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX= .. && make && make install ) + ( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX= .. && make && make install ) + +Please refer to the packages' documentation for more information about +the installation process. In particular, there's more information on +required and optional `dependencies for Lintel +`_ +and `dependencies for DataSeries +`_ + +Compiling Bro with DataSeries Support +------------------------------------- + +Once you have installed DataSeries, Bro's ``configure`` should pick it +up automatically as long as it finds it in a standard system location. +Alternatively, you can specify the DataSeries installation prefix +manually with ``--with-dataseries=``. Keep an eye on +``configure``'s summary output, if it looks like the following, Bro +found DataSeries and will compile in the support:: + + # ./configure --with-dataseries=/usr/local + [...] + ====================| Bro Build Summary |===================== + [...] + DataSeries: true + [...] + ================================================================ + +Activating DataSeries +--------------------- + +The direct way to use DataSeries is to switch *all* log files over to +the binary format. To do that, just add ``redef +Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``. +For testing, you can also just pass that on the command line:: + + bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES + +With that, Bro will now write all its output into DataSeries files +``*.ds``. You can inspect these using DataSeries's set of command line +tools, which its installation process installs into ``/bin``. +For example, to convert a file back into an ASCII representation:: + + $ ds2txt conn.log + [... We skip a bunch of meta data here ...] + ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes + 1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0 + 1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0 + 1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0 + 1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117 + 1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127 + 1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211 + [...] + +(``--skip-all`` suppresses the meta data.) + +Note that the ASCII conversion is *not* equivalent to Bro's default +output format. + +You can also switch only individual files over to DataSeries by adding +code like this to your ``local.bro``:: + +.. code:: bro + + event bro_init() + { + local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log. + f$writer = Log::WRITER_DATASERIES; # Change writer type. + Log::add_filter(Conn::LOG, f); # Replace filter with adapted version. + } + +Bro's DataSeries writer comes with a few tuning options, see +:doc:`scripts/base/frameworks/logging/writers/dataseries`. + +Working with DataSeries +======================= + +Here are few examples of using DataSeries command line tools to work +with the output files. + +* Printing CSV:: + + $ ds2txt --csv conn.log + ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes + 1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0 + 1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0 + 1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0 + 1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0 + [...] + + Add ``--separator=X`` to set a different separator. + +* Extracting a subset of columns:: + + $ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log + 1258790493.773208 192.168.1.255 137 + 1258790451.402091 192.168.1.255 138 + 1258790493.787448 192.168.1.255 138 + 1258790615.268111 192.168.1.255 137 + 1258790615.289842 192.168.1.255 138 + [...] + +* Filtering rows:: + + $ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all conn.ds + 1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp 15.004568 798 0 S0 F 0 D 6 966 0 0 + 1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp 15.004581 798 0 S0 F 0 D 6 966 0 0 + 1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp 15.005071 798 0 S0 F 0 D 6 966 0 0 + 1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp 15.004824 798 0 S0 F 0 D 6 966 0 0 + 1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp 15.005078 798 0 S0 F 0 D 6 966 0 0 + 1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp 15.004082 798 0 S0 F 0 D 6 966 0 0 + [...] + +* Calculate some statistics: + + Mean/stdev/min/max over a column:: + + $ dsstatgroupby '*' basic duration from conn.ds + # Begin DSStatGroupByModule + # processed 2159 rows, where clause eliminated 0 rows + # count(*), mean(duration), stddev, min, max + 2159, 42.7938, 1858.34, 0, 86370 + [...] + + Quantiles of total connection volume:: + + > dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds + [...] + 2159 data points, mean 24616 +- 343295 [0,1.26615e+07] + quantiles about every 216 data points: + 10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469 + tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262 + [...] + +The ``man`` pages for these tool show further options, and their +``-h`` option gives some more information (either can be a bit cryptic +unfortunately though). diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index ade0add875..1743b0258f 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -36,6 +36,7 @@ rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro) rest_target(${psd} base/frameworks/logging/writers/ascii.bro) +rest_target(${psd} base/frameworks/logging/writers/dataseries.bro) rest_target(${psd} base/frameworks/metrics/cluster.bro) rest_target(${psd} base/frameworks/metrics/main.bro) rest_target(${psd} base/frameworks/metrics/non-cluster.bro) diff --git a/scripts/base/frameworks/logging/__load__.bro b/scripts/base/frameworks/logging/__load__.bro index 42b2d7c564..17e03e2ef7 100644 --- a/scripts/base/frameworks/logging/__load__.bro +++ b/scripts/base/frameworks/logging/__load__.bro @@ -1,3 +1,4 @@ @load ./main @load ./postprocessors @load ./writers/ascii +@load ./writers/dataseries diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 2c36b3001e..bec5f31dc6 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -332,7 +332,7 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool function default_path_func(id: ID, path: string, rec: any) : string { local id_str = fmt("%s", id); - + local parts = split1(id_str, /::/); if ( |parts| == 2 ) { @@ -340,7 +340,7 @@ function default_path_func(id: ID, path: string, rec: any) : string # or a filter path explicitly set by the user, so continue using it. if ( path != "" ) return path; - + # Example: Notice::LOG -> "notice" if ( parts[2] == "LOG" ) { @@ -356,11 +356,11 @@ function default_path_func(id: ID, path: string, rec: any) : string output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|)); return to_lower(output); } - + # Example: Notice::POLICY_LOG -> "notice_policy" if ( /_LOG$/ in parts[2] ) parts[2] = sub(parts[2], /_LOG$/, ""); - + return cat(to_lower(parts[1]),"_",to_lower(parts[2])); } else @@ -376,13 +376,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo if ( pp_cmd == "" ) return T; + # Turn, e.g., Log::WRITER_ASCII into "ascii". + local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", ""); + # The date format is hard-coded here to provide a standardized # script interface. - system(fmt("%s %s %s %s %s %d", + system(fmt("%s %s %s %s %s %d %s", pp_cmd, npath, info$path, strftime("%y-%m-%d_%H.%M.%S", info$open), strftime("%y-%m-%d_%H.%M.%S", info$close), - info$terminating)); + info$terminating, writer)); return T; } @@ -407,7 +410,7 @@ function add_filter(id: ID, filter: Filter) : bool # definition. if ( ! filter?$path_func ) filter$path_func = default_path_func; - + filters[id, filter$name] = filter; return __add_filter(id, filter); } diff --git a/scripts/base/frameworks/logging/writers/dataseries.bro b/scripts/base/frameworks/logging/writers/dataseries.bro new file mode 100644 index 0000000000..ccee500c3a --- /dev/null +++ b/scripts/base/frameworks/logging/writers/dataseries.bro @@ -0,0 +1,60 @@ +##! Interface for the DataSeries log writer. + +module LogDataSeries; + +export { + ## Compression to use with the DS output file. Options are: + ## + ## 'none' -- No compression. + ## 'lzf' -- LZF compression. Very quick, but leads to larger output files. + ## 'lzo' -- LZO compression. Very fast decompression times. + ## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output. + ## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output. + const compression = "lzo" &redef; + + ## The extent buffer size. + ## Larger values here lead to better compression and more efficient writes, but + ## also increase the lag between the time events are received and the time they + ## are actually written to disk. + const extent_size = 65536 &redef; + + ## Should we dump the XML schema we use for this DS file to disk? + ## If yes, the XML schema shares the name of the logfile, but has + ## an XML ending. + const dump_schema = F &redef; + + ## How many threads should DataSeries spawn to perform compression? + ## Note that this dictates the number of threads per log stream. If + ## you're using a lot of streams, you may want to keep this number + ## relatively small. + ## + ## Default value is 1, which will spawn one thread / stream. + ## + ## Maximum is 128, minimum is 1. + const num_threads = 1 &redef; + + ## Should time be stored as an integer or a double? + ## Storing time as a double leads to possible precision issues and + ## can (significantly) increase the size of the resulting DS log. + ## That said, timestamps stored in double form are consistent + ## with the rest of Bro, including the standard ASCII log. Hence, we + ## use them by default. + const use_integer_for_time = F &redef; +} + +# Default function to postprocess a rotated DataSeries log file. It moves the +# rotated file to a new name that includes a timestamp with the opening time, and +# then runs the writer's default postprocessor command on it. +function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool + { + # Move file to name including both opening and closing time. + local dst = fmt("%s.%s.ds", info$path, + strftime(Log::default_rotation_date_format, info$open)); + + system(fmt("/bin/mv %s %s", info$fname, dst)); + + # Run default postprocessor. + return Log::run_rotation_postprocessor_cmd(info, dst); + } + +redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4f060d3bed..9bd7c1200c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -417,6 +417,7 @@ set(bro_SRCS logging/WriterBackend.cc logging/WriterFrontend.cc logging/writers/Ascii.cc + logging/writers/DataSeries.cc logging/writers/None.cc nb_dns.c diff --git a/src/Type.cc b/src/Type.cc index 82221303af..d688b15376 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -15,10 +15,9 @@ extern int generate_documentation; +// Note: This function must be thread-safe. const char* type_name(TypeTag t) { - static char errbuf[512]; - static const char* type_names[int(NUM_TYPES)] = { "void", "bool", "int", "count", "counter", @@ -37,10 +36,7 @@ const char* type_name(TypeTag t) }; if ( int(t) >= NUM_TYPES ) - { - snprintf(errbuf, sizeof(errbuf), "%d: not a type tag", int(t)); - return errbuf; - } + return "type_name(): not a type tag"; return type_names[int(t)]; } diff --git a/src/logging.bif b/src/logging.bif index c8960b4e38..efc6ed0b4b 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -72,3 +72,12 @@ const set_separator: string; const empty_field: string; const unset_field: string; +# Options for the DataSeries writer. + +module LogDataSeries; + +const compression: string; +const extent_size: count; +const dump_schema: bool; +const use_integer_for_time: bool; +const num_threads: count; diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 9437f0099f..f78e47da73 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -7,6 +7,7 @@ #include "../NetVar.h" #include "../Net.h" +#include "threading/Manager.h" #include "threading/SerialTypes.h" #include "Manager.h" @@ -16,9 +17,11 @@ #include "writers/Ascii.h" #include "writers/None.h" +#ifdef USE_DATASERIES +#include "writers/DataSeries.h" +#endif + using namespace logging; -using threading::Value; -using threading::Field; // Structure describing a log writer type. struct WriterDefinition { @@ -32,6 +35,9 @@ struct WriterDefinition { WriterDefinition log_writers[] = { { BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate }, { BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate }, +#ifdef USE_DATASERIES + { BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate }, +#endif // End marker, don't touch. { BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 } @@ -51,7 +57,7 @@ struct Manager::Filter { Func* postprocessor; int num_fields; - Field** fields; + threading::Field** fields; // Vector indexed by field number. Each element is a list of record // indices defining a path leading to the value across potential @@ -119,6 +125,7 @@ Manager::Stream::~Stream() Manager::Manager() { + rotations_pending = 0; } Manager::~Manager() @@ -127,6 +134,16 @@ Manager::~Manager() delete *s; } +list Manager::SupportedFormats() + { + list formats; + + for ( WriterDefinition* ld = log_writers; ld->type != BifEnum::Log::WRITER_DEFAULT; ++ld ) + formats.push_back(ld->name); + + return formats; + } + WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) { WriterDefinition* ld = log_writers; @@ -135,7 +152,7 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) { if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) { - reporter->Error("unknow writer when creating writer"); + reporter->Error("unknown writer type requested"); return 0; } @@ -159,10 +176,8 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type) // function. ld->factory = 0; - DBG_LOG(DBG_LOGGING, "failed to init writer class %s", - ld->name); - - return false; + reporter->Error("initialization of writer %s failed", ld->name); + return 0; } } @@ -449,7 +464,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, filter->indices.push_back(new_indices); - filter->fields = (Field**) + filter->fields = (threading::Field**) realloc(filter->fields, sizeof(Field) * ++filter->num_fields); @@ -459,7 +474,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, return false; } - Field* field = new Field(); + threading::Field* field = new threading::Field(); field->name = new_path; field->type = t->Tag(); @@ -571,7 +586,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval) for ( int i = 0; i < filter->num_fields; i++ ) { - Field* field = filter->fields[i]; + threading::Field* field = filter->fields[i]; DBG_LOG(DBG_LOGGING, " field %10s: %s", field->name.c_str(), type_name(field->type)); } @@ -743,10 +758,10 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) // Copy the fields for WriterFrontend::Init() as it // will take ownership. - Field** arg_fields = new Field*[filter->num_fields]; + threading::Field** arg_fields = new threading::Field*[filter->num_fields]; for ( int j = 0; j < filter->num_fields; ++j ) - arg_fields[j] = new Field(*filter->fields[j]); + arg_fields[j] = new threading::Field(*filter->fields[j]); writer = CreateWriter(stream->id, filter->writer, path, filter->num_fields, @@ -897,10 +912,10 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) return lval; } -Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, +threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns) { - Value** vals = new Value*[filter->num_fields]; + threading::Value** vals = new threading::Value*[filter->num_fields]; for ( int i = 0; i < filter->num_fields; ++i ) { @@ -919,7 +934,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, if ( ! val ) { // Value, or any of its parents, is not set. - vals[i] = new Value(filter->fields[i]->type, false); + vals[i] = new threading::Value(filter->fields[i]->type, false); break; } } @@ -932,7 +947,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, } WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const Field* const* fields, bool local, bool remote) + int num_fields, const threading::Field* const* fields, bool local, bool remote) { Stream* stream = FindStream(id); @@ -996,7 +1011,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, return writer_obj; } -void Manager::DeleteVals(int num_fields, Value** vals) +void Manager::DeleteVals(int num_fields, threading::Value** vals) { // Note this code is duplicated in WriterBackend::DeleteVals(). for ( int i = 0; i < num_fields; i++ ) @@ -1006,7 +1021,7 @@ void Manager::DeleteVals(int num_fields, Value** vals) } bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, - Value** vals) + threading::Value** vals) { Stream* stream = FindStream(id); @@ -1113,10 +1128,19 @@ bool Manager::Flush(EnumVal* id) void Manager::Terminate() { + // Make sure we process all the pending rotations. + while ( rotations_pending ) + { + thread_mgr->ForceProcessing(); // A blatant layering violation ... + usleep(1000); + } + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) { - if ( *s ) - Flush((*s)->id); + if ( ! *s ) + continue; + + Flush((*s)->id); } } @@ -1219,11 +1243,19 @@ void Manager::Rotate(WriterInfo* winfo) // Trigger the rotation. winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating); + + ++rotations_pending; } bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name, double open, double close, bool terminating) { + --rotations_pending; + + if ( ! writer ) + // Writer didn't produce local output. + return true; + DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s", writer->Path().c_str(), network_time, new_name.c_str()); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index bf097c5e1a..f5e62b0683 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -15,7 +15,6 @@ class RotationTimer; namespace logging { - class WriterBackend; class WriterFrontend; class RotationFinishedMessage; @@ -56,7 +55,7 @@ public: * logging.bif, which just forwards here. */ bool EnableStream(EnumVal* id); - + /** * Disables a log stream. * @@ -145,6 +144,11 @@ public: */ void Terminate(); + /** + * Returns a list of supported output formats. + */ + static list SupportedFormats(); + protected: friend class WriterFrontend; friend class RotationFinishedMessage; @@ -196,6 +200,7 @@ private: WriterInfo* FindWriter(WriterFrontend* writer); vector streams; // Indexed by stream enum. + int rotations_pending; // Number of rotations not yet finished. }; } diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index c33f6a285a..23a95279d7 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -223,17 +223,6 @@ bool WriterBackend::Flush() return true; } -bool WriterBackend::Finish() - { - if ( ! DoFlush() ) - { - DisableFrontend(); - return false; - } - - return true; - } - bool WriterBackend::DoHeartbeat(double network_time, double current_time) { MsgThread::DoHeartbeat(network_time, current_time); @@ -279,4 +268,9 @@ string WriterBackend::Render(const threading::Value::subnet_t& subnet) const return s; } - +string WriterBackend::Render(double d) const + { + char buf[256]; + modp_dtoa(d, buf, 6); + return buf; + } diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index eea0927fd2..1269976aee 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -101,15 +101,6 @@ public: */ bool Rotate(string rotated_path, double open, double close, bool terminating); - /** - * Finishes writing to this logger in a regularl fashion. Must not be - * called if an error has been indicated earlier. After calling this, - * no further writing must be performed. - * - * @return False if an error occured. - */ - bool Finish(); - /** * Disables the frontend that has instantiated this backend. Once * disabled,the frontend will not send any further message over. @@ -174,7 +165,17 @@ public: */ string Render(const threading::Value::subnet_t& subnet) const; + /** Helper method to render a double in Bro's standard precision. + * + * @param d The double. + * + * @return An ASCII representation of the double. + */ + string Render(double d) const; + protected: + friend class FinishMessage; + /** * Writer-specific intialization method. * @@ -272,26 +273,18 @@ protected: bool terminating) = 0; /** - * Writer-specific method implementing log output finalization at - * termination. Not called when any of the other methods has - * previously signaled an error, i.e., executing this method signals - * a regular shutdown of the writer. + * Writer-specific method called just before the threading system is + * going to shutdown. * - * A writer implementation must override this method but it can just - * ignore calls if flushing doesn't align with its semantics. - * - * If the method returns false, it will be assumed that a fatal error - * has occured that prevents the writer from further operation; it - * will then be disabled and eventually deleted. When returning - * false, an implementation should also call Error() to indicate what - * happened. + * This method can be overridden but one must call + * WriterBackend::DoFinish(). */ - virtual bool DoFinish() = 0; + virtual bool DoFinish() { return MsgThread::DoFinish(); } /** * Triggered by regular heartbeat messages from the main thread. * - * This method can be overridden but once must call + * This method can be overridden but one must call * WriterBackend::DoHeartbeat(). */ virtual bool DoHeartbeat(double network_time, double current_time); diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index c100e99f90..33c9c04c63 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -90,7 +90,7 @@ public: FinishMessage(WriterBackend* backend) : threading::InputMessage("Finish", backend) {} - virtual bool Process() { return Object()->Finish(); } + virtual bool Process() { return Object()->DoFinish(); } }; } @@ -117,8 +117,9 @@ WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool ar if ( local ) { backend = log_mgr->CreateBackend(this, writer->AsEnum()); - assert(backend); - backend->Start(); + + if ( backend ) + backend->Start(); } else @@ -256,6 +257,10 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool if ( backend ) backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + else + // Still signal log manager that we're done, but signal that + // nothing happened by setting the writer to zeri. + log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating); } void WriterFrontend::Finish() diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index 0759e60a82..1e7a55c34c 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -69,8 +69,7 @@ bool Ascii::WriteHeaderField(const string& key, const string& val) return (fwrite(str.c_str(), str.length(), 1, file) == 1); } -bool Ascii::DoInit(string path, int num_fields, - const Field* const * fields) +bool Ascii::DoInit(string path, int num_fields, const Field* const * fields) { if ( output_to_stdout ) path = "/dev/stdout"; @@ -87,6 +86,9 @@ bool Ascii::DoInit(string path, int num_fields, if ( include_header ) { + string names; + string types; + string str = string(header_prefix, header_prefix_len) + "separator " // Always use space as separator here. + get_escaped_string(string(separator, separator_len), false) @@ -104,9 +106,6 @@ bool Ascii::DoInit(string path, int num_fields, WriteHeaderField("path", get_escaped_string(path, false))) ) goto write_error; - string names; - string types; - for ( int i = 0; i < num_fields; ++i ) { if ( i > 0 ) @@ -115,15 +114,8 @@ bool Ascii::DoInit(string path, int num_fields, types += string(separator, separator_len); } - const Field* field = fields[i]; - names += field->name; - types += type_name(field->type); - if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) ) - { - types += "["; - types += type_name(field->subtype); - types += "]"; - } + names += fields[i]->name; + types += fields[i]->TypeName(); } if ( ! (WriteHeaderField("fields", names) @@ -146,7 +138,7 @@ bool Ascii::DoFlush() bool Ascii::DoFinish() { - return true; + return WriterBackend::DoFinish(); } bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) @@ -184,15 +176,19 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field) desc->Add(Render(val->val.addr_val)); break; - case TYPE_TIME: - case TYPE_INTERVAL: - char buf[256]; - modp_dtoa(val->val.double_val, buf, 6); - desc->Add(buf); + case TYPE_DOUBLE: + // Rendering via Add() truncates trailing 0s after the + // decimal point. The difference with TIME/INTERVAL is mainly + // to keep the log format consistent. + desc->Add(val->val.double_val); break; - case TYPE_DOUBLE: - desc->Add(val->val.double_val); + case TYPE_INTERVAL: + case TYPE_TIME: + // Rendering via Render() keeps trailing 0s after the decimal + // point. The difference with DOUBLEis mainly to keep the log + // format consistent. + desc->Add(Render(val->val.double_val)); break; case TYPE_ENUM: diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc new file mode 100644 index 0000000000..9f19028be3 --- /dev/null +++ b/src/logging/writers/DataSeries.cc @@ -0,0 +1,417 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include +#include + +#include + +#include "NetVar.h" +#include "threading/SerialTypes.h" + +#include "DataSeries.h" + +using namespace logging; +using namespace writer; + +std::string DataSeries::LogValueToString(threading::Value *val) + { + // In some cases, no value is attached. If this is the case, return + // an empty string. + if( ! val->present ) + return ""; + + switch(val->type) { + case TYPE_BOOL: + return (val->val.int_val ? "true" : "false"); + + case TYPE_INT: + { + std::ostringstream ostr; + ostr << val->val.int_val; + return ostr.str(); + } + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + { + std::ostringstream ostr; + ostr << val->val.uint_val; + return ostr.str(); + } + + case TYPE_SUBNET: + return Render(val->val.subnet_val); + + case TYPE_ADDR: + return Render(val->val.addr_val); + + // Note: These two cases are relatively special. We need to convert + // these values into their integer equivalents to maximize precision. + // At the moment, there won't be a noticeable effect (Bro uses the + // double format everywhere internally, so we've already lost the + // precision we'd gain here), but timestamps may eventually switch to + // this representation within Bro. + // + // In the near-term, this *should* lead to better pack_relative (and + // thus smaller output files). + case TYPE_TIME: + case TYPE_INTERVAL: + if ( ds_use_integer_for_time ) + { + std::ostringstream ostr; + ostr << (uint64_t)(DataSeries::TIME_SCALE * val->val.double_val); + return ostr.str(); + } + else + return Render(val->val.double_val); + + case TYPE_DOUBLE: + return Render(val->val.double_val); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + if ( ! val->val.string_val->size() ) + return ""; + + return string(val->val.string_val->data(), val->val.string_val->size()); + + case TYPE_TABLE: + { + if ( ! val->val.set_val.size ) + return ""; + + string tmpString = ""; + + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + tmpString += ds_set_separator; + + tmpString += LogValueToString(val->val.set_val.vals[j]); + } + + return tmpString; + } + + case TYPE_VECTOR: + { + if ( ! val->val.vector_val.size ) + return ""; + + string tmpString = ""; + + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + tmpString += ds_set_separator; + + tmpString += LogValueToString(val->val.vector_val.vals[j]); + } + + return tmpString; + } + + default: + InternalError(Fmt("unknown type %s in DataSeries::LogValueToString", type_name(val->type))); + return "cannot be reached"; + } +} + +string DataSeries::GetDSFieldType(const threading::Field *field) +{ + switch(field->type) { + case TYPE_BOOL: + return "bool"; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_INT: + return "int64"; + + case TYPE_DOUBLE: + return "double"; + + case TYPE_TIME: + case TYPE_INTERVAL: + return ds_use_integer_for_time ? "int64" : "double"; + + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_TABLE: + case TYPE_VECTOR: + case TYPE_FUNC: + return "variable32"; + + default: + InternalError(Fmt("unknown type %s in DataSeries::GetDSFieldType", type_name(field->type))); + return "cannot be reached"; + } +} + +string DataSeries::BuildDSSchemaFromFieldTypes(const vector& vals, string sTitle) + { + if( ! sTitle.size() ) + sTitle = "GenericBroStream"; + + string xmlschema = "\n"; + + for( size_t i = 0; i < vals.size(); ++i ) + { + xmlschema += "\t\n"; + } + + xmlschema += "\n"; + + for( size_t i = 0; i < vals.size(); ++i ) + { + xmlschema += "\n"; + } + + return xmlschema; +} + +std::string DataSeries::GetDSOptionsForType(const threading::Field *field) +{ + switch( field->type ) { + case TYPE_TIME: + case TYPE_INTERVAL: + { + std::string s; + s += "pack_relative=\"" + std::string(field->name) + "\""; + + if ( ! ds_use_integer_for_time ) + s += " pack_scale=\"1e-6\" print_format=\"%.6f\" pack_scale_warn=\"no\""; + else + s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\""; + + return s; + } + + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + case TYPE_TABLE: + case TYPE_VECTOR: + return "pack_unique=\"yes\""; + + default: + return ""; + } +} + +DataSeries::DataSeries(WriterFrontend* frontend) : WriterBackend(frontend) +{ + ds_compression = string((const char *)BifConst::LogDataSeries::compression->Bytes(), + BifConst::LogDataSeries::compression->Len()); + ds_dump_schema = BifConst::LogDataSeries::dump_schema; + ds_extent_size = BifConst::LogDataSeries::extent_size; + ds_num_threads = BifConst::LogDataSeries::num_threads; + ds_use_integer_for_time = BifConst::LogDataSeries::use_integer_for_time; + ds_set_separator = ","; +} + +DataSeries::~DataSeries() +{ +} + +bool DataSeries::OpenLog(string path) + { + log_file = new DataSeriesSink(path + ".ds", compress_type); + log_file->writeExtentLibrary(log_types); + + for( size_t i = 0; i < schema_list.size(); ++i ) + extents.insert(std::make_pair(schema_list[i].field_name, + GeneralField::create(log_series, schema_list[i].field_name))); + + if ( ds_extent_size < ROW_MIN ) + { + Warning(Fmt("%d is not a valid value for 'rows'. Using min of %d instead", (int)ds_extent_size, (int)ROW_MIN)); + ds_extent_size = ROW_MIN; + } + + else if( ds_extent_size > ROW_MAX ) + { + Warning(Fmt("%d is not a valid value for 'rows'. Using max of %d instead", (int)ds_extent_size, (int)ROW_MAX)); + ds_extent_size = ROW_MAX; + } + + log_output = new OutputModule(*log_file, log_series, log_type, ds_extent_size); + + return true; + } + +bool DataSeries::DoInit(string path, int num_fields, const threading::Field* const * fields) + { + // We first construct an XML schema thing (and, if ds_dump_schema is + // set, dump it to path + ".ds.xml"). Assuming that goes well, we + // use that schema to build our output logfile and prepare it to be + // written to. + + // Note: compressor count must be set *BEFORE* DataSeriesSink is + // instantiated. + if( ds_num_threads < THREAD_MIN && ds_num_threads != 0 ) + { + Warning(Fmt("%d is too few threads! Using %d instead", (int)ds_num_threads, (int)THREAD_MIN)); + ds_num_threads = THREAD_MIN; + } + + if( ds_num_threads > THREAD_MAX ) + { + Warning(Fmt("%d is too many threads! Dropping back to %d", (int)ds_num_threads, (int)THREAD_MAX)); + ds_num_threads = THREAD_MAX; + } + + if( ds_num_threads > 0 ) + DataSeriesSink::setCompressorCount(ds_num_threads); + + for ( int i = 0; i < num_fields; i++ ) + { + const threading::Field* field = fields[i]; + SchemaValue val; + val.ds_type = GetDSFieldType(field); + val.field_name = string(field->name); + val.field_options = GetDSOptionsForType(field); + val.bro_type = field->TypeName(); + schema_list.push_back(val); + } + + string schema = BuildDSSchemaFromFieldTypes(schema_list, path); + + if( ds_dump_schema ) + { + FILE* pFile = fopen ( string(path + ".ds.xml").c_str() , "wb" ); + + if( pFile ) + { + fwrite(schema.c_str(), 1, schema.length(), pFile); + fclose(pFile); + } + + else + Error(Fmt("cannot dump schema: %s", strerror(errno))); + } + + compress_type = Extent::compress_all; + + if( ds_compression == "lzf" ) + compress_type = Extent::compress_lzf; + + else if( ds_compression == "lzo" ) + compress_type = Extent::compress_lzo; + + else if( ds_compression == "gz" ) + compress_type = Extent::compress_gz; + + else if( ds_compression == "bz2" ) + compress_type = Extent::compress_bz2; + + else if( ds_compression == "none" ) + compress_type = Extent::compress_none; + + else if( ds_compression == "any" ) + compress_type = Extent::compress_all; + + else + Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str())); + + log_type = log_types.registerTypePtr(schema); + log_series.setType(log_type); + + return OpenLog(path); + } + +bool DataSeries::DoFlush() +{ + // Flushing is handled by DataSeries automatically, so this function + // doesn't do anything. + return true; +} + +void DataSeries::CloseLog() + { + for( ExtentIterator iter = extents.begin(); iter != extents.end(); ++iter ) + delete iter->second; + + extents.clear(); + + // Don't delete the file before you delete the output, or bad things + // will happen. + delete log_output; + delete log_file; + + log_output = 0; + log_file = 0; + } + +bool DataSeries::DoFinish() +{ + CloseLog(); + + return WriterBackend::DoFinish(); +} + +bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields, + threading::Value** vals) +{ + log_output->newRecord(); + + for( size_t i = 0; i < (size_t)num_fields; ++i ) + { + ExtentIterator iter = extents.find(fields[i]->name); + assert(iter != extents.end()); + + if( iter != extents.end() ) + { + GeneralField *cField = iter->second; + + if( vals[i]->present ) + cField->set(LogValueToString(vals[i])); + } + } + + return true; +} + +bool DataSeries::DoRotate(string rotated_path, double open, double close, bool terminating) +{ + // Note that if DS files are rotated too often, the aggregate log + // size will be (much) larger. + CloseLog(); + + string dsname = Path() + ".ds"; + string nname = rotated_path + ".ds"; + rename(dsname.c_str(), nname.c_str()); + + if ( ! FinishedRotation(nname, dsname, open, close, terminating) ) + { + Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str())); + return false; + } + + return OpenLog(Path()); +} + +bool DataSeries::DoSetBuf(bool enabled) +{ + // DataSeries is *always* buffered to some degree. This option is ignored. + return true; +} diff --git a/src/logging/writers/DataSeries.h b/src/logging/writers/DataSeries.h new file mode 100644 index 0000000000..0d9ab67e95 --- /dev/null +++ b/src/logging/writers/DataSeries.h @@ -0,0 +1,124 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// A binary log writer producing DataSeries output. See doc/data-series.rst +// for more information. + +#ifndef LOGGING_WRITER_DATA_SERIES_H +#define LOGGING_WRITER_DATA_SERIES_H + +#include +#include +#include +#include + +#include "../WriterBackend.h" + +namespace logging { namespace writer { + +class DataSeries : public WriterBackend { +public: + DataSeries(WriterFrontend* frontend); + ~DataSeries(); + + static WriterBackend* Instantiate(WriterFrontend* frontend) + { return new DataSeries(frontend); } + +protected: + // Overidden from WriterBackend. + + virtual bool DoInit(string path, int num_fields, + const threading::Field* const * fields); + + virtual bool DoWrite(int num_fields, const threading::Field* const* fields, + threading::Value** vals); + virtual bool DoSetBuf(bool enabled); + virtual bool DoRotate(string rotated_path, double open, + double close, bool terminating); + virtual bool DoFlush(); + virtual bool DoFinish(); + +private: + static const size_t ROW_MIN = 2048; // Minimum extent size. + static const size_t ROW_MAX = (1024 * 1024 * 100); // Maximum extent size. + static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn. + static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn. + static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers. + const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE. + + struct SchemaValue + { + string ds_type; + string bro_type; + string field_name; + string field_options; + }; + + /** + * Turns a log value into a std::string. Uses an ostringstream to do the + * heavy lifting, but still need to switch on the type to know which value + * in the union to give to the string string for processing. + * + * @param val The value we wish to convert to a string + * @return the string value of val + */ + std::string LogValueToString(threading::Value *val); + + /** + * Takes a field type and converts it to a relevant DataSeries type. + * + * @param field We extract the type from this and convert it into a relevant DS type. + * @return String representation of type that DataSeries can understand. + */ + string GetDSFieldType(const threading::Field *field); + + /** + * Are there any options we should put into the XML schema? + * + * @param field We extract the type from this and return any options that make sense for that type. + * @return Options that can be added directly to the XML (e.g. "pack_relative=\"yes\"") + */ + std::string GetDSOptionsForType(const threading::Field *field); + + /** + * Takes a list of types, a list of names, and a title, and uses it to construct a valid DataSeries XML schema + * thing, which is then returned as a std::string + * + * @param opts std::vector of strings containing a list of options to be appended to each field (e.g. "pack_relative=yes") + * @param sTitle Name of this schema. Ideally, these schemas would be aggregated and re-used. + */ + string BuildDSSchemaFromFieldTypes(const vector& vals, string sTitle); + + /** Closes the currently open file. */ + void CloseLog(); + + /** Opens a new file. */ + bool OpenLog(string path); + + typedef std::map ExtentMap; + typedef ExtentMap::iterator ExtentIterator; + + // Internal DataSeries structures we need to keep track of. + vector schema_list; + ExtentTypeLibrary log_types; + ExtentType::Ptr log_type; + ExtentSeries log_series; + ExtentMap extents; + int compress_type; + + DataSeriesSink* log_file; + OutputModule* log_output; + + // Options set from the script-level. + uint64 ds_extent_size; + uint64 ds_num_threads; + string ds_compression; + bool ds_dump_schema; + bool ds_use_integer_for_time; + string ds_set_separator; +}; + +} +} + +#endif + diff --git a/src/main.cc b/src/main.cc index 89783031bf..19910aebc5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -203,6 +203,27 @@ void usage() fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " Supported log formats: "); + + bool first = true; + list fmts = logging::Manager::SupportedFormats(); + + for ( list::const_iterator i = fmts.begin(); i != fmts.end(); ++i ) + { + if ( *i == "None" ) + // Skip, it's uninteresting. + continue; + + if ( ! first ) + fprintf(stderr, ","); + + fprintf(stderr, "%s", (*i).c_str()); + first = false; + } + + fprintf(stderr, "\n"); + exit(1); } diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 7d9ba766d4..ab8189f39d 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -77,6 +77,12 @@ public: */ int NumThreads() const { return all_threads.size(); } + /** Manually triggers processing of any thread input. This can be useful + * if the main thread is waiting for a specific message from a child. + * Usually, though, one should avoid using it. + */ + void ForceProcessing() { Process(); } + protected: friend class BasicThread; friend class MsgThread; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index ddcd3df1dd..dd73fae154 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -10,13 +10,21 @@ namespace threading { ////// Messages. -// Signals child thread to terminate. This is actually a no-op; its only -// purpose is unblock the current read operation so that the child's Run() -// methods can check the termination status. -class TerminateMessage : public InputMessage +// Signals child thread to shutdown operation. +class FinishMessage : public InputMessage { public: - TerminateMessage(MsgThread* thread) : InputMessage("Terminate", thread) { } + FinishMessage(MsgThread* thread) : InputMessage("Finish", thread) { } + + virtual bool Process() { return Object()->DoFinish(); } +}; + +// A dummy message that's only purpose is unblock the current read operation +// so that the child's Run() methods can check the termination status. +class UnblockMessage : public InputMessage +{ +public: + UnblockMessage(MsgThread* thread) : InputMessage("Unblock", thread) { } virtual bool Process() { return true; } }; @@ -130,13 +138,29 @@ bool ReporterMessage::Process() MsgThread::MsgThread() : BasicThread() { cnt_sent_in = cnt_sent_out = 0; + finished = false; thread_mgr->AddMsgThread(this); } void MsgThread::OnStop() { - // This is to unblock the current queue read operation. - SendIn(new TerminateMessage(this), true); + // Signal thread to terminate and wait until it has acknowledged. + SendIn(new FinishMessage(this), true); + + int cnt = 0; + while ( ! finished ) + { + if ( ++cnt > 1000 ) // Insurance against broken threads ... + { + reporter->Warning("thread %s didn't finish in time", Name().c_str()); + break; + } + + usleep(1000); + } + + // One more message to make sure the current queue read operation unblocks. + SendIn(new UnblockMessage(this), true); } void MsgThread::Heartbeat() @@ -157,6 +181,14 @@ bool MsgThread::DoHeartbeat(double network_time, double current_time) return true; } +bool MsgThread::DoFinish() + { + // This is thread-safe "enough", we're the only one ever writing + // there. + finished = true; + return true; + } + void MsgThread::Info(const char* msg) { SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg)); @@ -189,7 +221,9 @@ void MsgThread::InternalWarning(const char* msg) void MsgThread::InternalError(const char* msg) { - SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg)); + // This one aborts immediately. + fprintf(stderr, "internal error in thread: %s\n", msg); + abort(); } #ifdef DEBUG diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 5ac1c0f780..cd29fe2a44 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -171,6 +171,8 @@ public: protected: friend class Manager; friend class HeartbeatMessage; + friend class FinishMessage; + friend class FinishedMessage; /** * Pops a message sent by the child from the child-to-main queue. @@ -215,6 +217,12 @@ protected: */ virtual bool DoHeartbeat(double network_time, double current_time); + /** Triggered for execution in the child thread just before shutting threads down. + * The child thread should finish its operations and then *must* + * call this class' implementation. + */ + virtual bool DoFinish(); + private: /** * Pops a message sent by the main thread from the main-to-chold @@ -270,6 +278,8 @@ private: uint64_t cnt_sent_in; // Counts message sent to child. uint64_t cnt_sent_out; // Counts message sent by child. + + bool finished; // Set to true by Finished message. }; /** diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index a5692b2ffd..5ab61b0d41 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -24,6 +24,20 @@ bool Field::Write(SerializationFormat* fmt) const return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } +string Field::TypeName() const + { + string n = type_name(type); + + if ( (type == TYPE_TABLE) || (type == TYPE_VECTOR) ) + { + n += "["; + n += type_name(subtype); + n += "]"; + } + + return n; + } + Value::~Value() { if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC) diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index db7dc837bd..eee3b750fe 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -53,6 +53,12 @@ struct Field { * @return False if an error occured. */ bool Write(SerializationFormat* fmt) const; + + /** + * Returns a textual description of the field's type. This method is + * thread-safe. + */ + string TypeName() const; }; /** @@ -132,8 +138,8 @@ struct Value { /** * Returns true if the type can be represented by a Value. If - * `atomic_only` is true, will not permit composite types. - */ + * `atomic_only` is true, will not permit composite types. This + * method is thread-safe. */ static bool IsCompatibleType(BroType* t, bool atomic_only=false); private: diff --git a/src/types.bif b/src/types.bif index 4657584a90..fe2e6ff861 100644 --- a/src/types.bif +++ b/src/types.bif @@ -162,6 +162,7 @@ enum Writer %{ WRITER_DEFAULT, WRITER_NONE, WRITER_ASCII, + WRITER_DATASERIES, %} enum ID %{ diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index d43367f300..2936e3b698 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -19,4 +19,5 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./postprocessors/./sftp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/logging/./writers/dataseries.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 92deb62edb..b2a31beafd 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -19,6 +19,7 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./postprocessors/./sftp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/logging/./writers/dataseries.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/./patterns.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml new file mode 100644 index 0000000000..cacc3b0ea4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out new file mode 100644 index 0000000000..1e5e1b05c6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out @@ -0,0 +1,290 @@ +test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries +test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries +test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries +test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries +test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries +test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries +test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries +test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries +test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries +test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries +> test.2011-03-07-03-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.000000 10.0.0.1 20 10.0.0.2 1024 +1299470395.000000 10.0.0.2 20 10.0.0.3 0 +> test.2011-03-07-04-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.000000 10.0.0.1 20 10.0.0.2 1025 +1299473995.000000 10.0.0.2 20 10.0.0.3 1 +> test.2011-03-07-05-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.000000 10.0.0.1 20 10.0.0.2 1026 +1299477595.000000 10.0.0.2 20 10.0.0.3 2 +> test.2011-03-07-06-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.000000 10.0.0.1 20 10.0.0.2 1027 +1299481195.000000 10.0.0.2 20 10.0.0.3 3 +> test.2011-03-07-07-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.000000 10.0.0.1 20 10.0.0.2 1028 +1299484795.000000 10.0.0.2 20 10.0.0.3 4 +> test.2011-03-07-08-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.000000 10.0.0.1 20 10.0.0.2 1029 +1299488395.000000 10.0.0.2 20 10.0.0.3 5 +> test.2011-03-07-09-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.000000 10.0.0.1 20 10.0.0.2 1030 +1299491995.000000 10.0.0.2 20 10.0.0.3 6 +> test.2011-03-07-10-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.000000 10.0.0.1 20 10.0.0.2 1031 +1299495595.000000 10.0.0.2 20 10.0.0.3 7 +> test.2011-03-07-11-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.000000 10.0.0.1 20 10.0.0.2 1032 +1299499195.000000 10.0.0.2 20 10.0.0.3 8 +> test.2011-03-07-12-00-05.ds +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='test' +t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.000000 10.0.0.1 20 10.0.0.2 1033 +1299502795.000000 10.0.0.2 20 10.0.0.3 9 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt new file mode 100644 index 0000000000..e9640dfd9d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt @@ -0,0 +1,34 @@ +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='ssh' +t id.orig_h id.orig_p id.resp_h id.resp_p status country +1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success unknown +1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure US +1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure UK +1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success BR +1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt new file mode 100644 index 0000000000..1d7cba3b3c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt @@ -0,0 +1,87 @@ +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='conn' +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes +1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 +1300475167097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0 +1300475167099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0 +1300475168853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 435 0 89 SHR F 0 Cd 0 0 1 117 +1300475168854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 420 0 99 SHR F 0 Cd 0 0 1 127 +1300475168854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 391 0 183 SHR F 0 Cd 0 0 1 211 +1300475168857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 317 0 89 SHR F 0 Cd 0 0 1 117 +1300475168858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 343 0 99 SHR F 0 Cd 0 0 1 127 +1300475168858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 375 0 183 SHR F 0 Cd 0 0 1 211 +1300475168891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 339 0 89 SHR F 0 Cd 0 0 1 117 +1300475168892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 334 0 99 SHR F 0 Cd 0 0 1 127 +1300475168892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 420 0 183 SHR F 0 Cd 0 0 1 211 +1300475168893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 384 0 89 SHR F 0 Cd 0 0 1 117 +1300475168894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 316 0 99 SHR F 0 Cd 0 0 1 127 +1300475168894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 422 0 183 SHR F 0 Cd 0 0 1 211 +1300475168901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 402 0 131 SHR F 0 Cd 0 0 1 159 +1300475168902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 374 0 198 SHR F 0 Cd 0 0 1 226 +1300475169899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0 +1300475170862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2613016 350 0 S0 F 0 D 7 546 0 0 +1300475171675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 100096 66 0 S0 F 0 D 2 162 0 0 +1300475171677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 100020 66 0 S0 F 0 D 2 122 0 0 +1300475173116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 99801 66 0 S0 F 0 D 2 162 0 0 +1300475173117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 99848 66 0 S0 F 0 D 2 122 0 0 +1300475173153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0 +1300475168859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 215893 1130 734 S1 F 1130 ShACad 4 216 4 950 +1300475168652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 61328 0 350 OTH F 0 CdA 1 52 1 402 +1300475168895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 227283 1178 734 S1 F 1178 ShACad 4 216 4 950 +1300475168902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 120040 534 412 S1 F 534 ShACad 3 164 3 576 +1300475168892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 229603 1148 734 S1 F 1148 ShACad 4 216 4 950 +1300475168855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 218501 1171 733 S1 F 1171 ShACad 4 216 4 949 +1300475168892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 220960 1137 733 S1 F 1137 ShACad 4 216 4 949 +1300475169780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48 +1300475168724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 119904 525 232 S1 F 525 ShACad 3 164 3 396 +1300475168855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 219720 1125 734 S1 F 1125 ShACad 4 216 4 950 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt new file mode 100644 index 0000000000..3cafa078de --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt @@ -0,0 +1,87 @@ +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='conn' +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes +1300475167.096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0 +1300475167.097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0 +1300475167.099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0 +1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 0 89 SHR F 0 Cd 0 0 1 117 +1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 0 99 SHR F 0 Cd 0 0 1 127 +1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 0 183 SHR F 0 Cd 0 0 1 211 +1300475168.857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 0 89 SHR F 0 Cd 0 0 1 117 +1300475168.858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 0 99 SHR F 0 Cd 0 0 1 127 +1300475168.858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 0 183 SHR F 0 Cd 0 0 1 211 +1300475168.891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 0 89 SHR F 0 Cd 0 0 1 117 +1300475168.892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 0 99 SHR F 0 Cd 0 0 1 127 +1300475168.892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 0 183 SHR F 0 Cd 0 0 1 211 +1300475168.893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 0 89 SHR F 0 Cd 0 0 1 117 +1300475168.894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 0 99 SHR F 0 Cd 0 0 1 127 +1300475168.894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 0 183 SHR F 0 Cd 0 0 1 211 +1300475168.901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 0 131 SHR F 0 Cd 0 0 1 159 +1300475168.902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 0 198 SHR F 0 Cd 0 0 1 226 +1300475169.899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0 +1300475170.862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0 +1300475171.675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0 +1300475171.677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 F 0 D 2 122 0 0 +1300475173.116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0 +1300475173.117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0 +1300475173.153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0 +1300475168.859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0.215893 1130 734 S1 F 1130 ShACad 4 216 4 950 +1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 0 350 OTH F 0 CdA 1 52 1 402 +1300475168.895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0.227284 1178 734 S1 F 1178 ShACad 4 216 4 950 +1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0.120041 534 412 S1 F 534 ShACad 3 164 3 576 +1300475168.892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0.229603 1148 734 S1 F 1148 ShACad 4 216 4 950 +1300475168.855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0.218501 1171 733 S1 F 1171 ShACad 4 216 4 949 +1300475168.892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0.220961 1137 733 S1 F 1137 ShACad 4 216 4 949 +1300475169.780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48 +1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0.119905 525 232 S1 F 525 ShACad 3 164 3 396 +1300475168.855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0.219720 1125 734 S1 F 1125 ShACad 4 216 4 950 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt new file mode 100644 index 0000000000..adb7bb3f7b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt @@ -0,0 +1,81 @@ +# Extent Types ... + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Extent, type='http' +ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file +1300475168.843894 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0 +1300475168.975800 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475168.976327 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475168.979160 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.012666 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.012730 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.014860 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.022665 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0 +1300475169.036294 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.036798 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.039923 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.074793 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.074938 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475169.075065 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr index 0954137b7e..e1958d67ad 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr @@ -1,10 +1,10 @@ -1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 -1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 -1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 -1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 -1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 -1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 -1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 -1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 -1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 -1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 +1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii +1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii +1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii +1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii +1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii +1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii +1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii +1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii +1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii +1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out index d31783edc4..c335b5eeb9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out @@ -1,13 +1,13 @@ -test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 -test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 -test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 -test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 -test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 -test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 -test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 -test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 -test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 -test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 +test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii +test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii +test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii +test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii +test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii +test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii +test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii +test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii +test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii +test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii > test.2011-03-07-03-00-05.log #separator \x09 #set_separator , diff --git a/testing/btest/scripts/base/frameworks/logging/rotation.trace b/testing/btest/Traces/rotation.trace similarity index 100% rename from testing/btest/scripts/base/frameworks/logging/rotation.trace rename to testing/btest/Traces/rotation.trace diff --git a/testing/btest/core/leaks/dataseries-rotate.bro b/testing/btest/core/leaks/dataseries-rotate.bro new file mode 100644 index 0000000000..6a3b5550cc --- /dev/null +++ b/testing/btest/core/leaks/dataseries-rotate.bro @@ -0,0 +1,35 @@ +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-GROUP: leaks +# @TEST-GROUP: dataseries +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/rotation.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { LOG }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + } &log; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor_cmd = "echo"; + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); +} + +event new_connection(c: connection) + { + Log::write(Test::LOG, [$t=network_time(), $id=c$id]); + } diff --git a/testing/btest/core/leaks/dataseries.bro b/testing/btest/core/leaks/dataseries.bro new file mode 100644 index 0000000000..b72b880612 --- /dev/null +++ b/testing/btest/core/leaks/dataseries.bro @@ -0,0 +1,10 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-GROUP: leaks +# @TEST-GROUP: dataseries +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro new file mode 100644 index 0000000000..fc3752a168 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro @@ -0,0 +1,44 @@ +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries +# +# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES +# @TEST-EXEC: test -e ssh.ds.xml +# @TEST-EXEC: btest-diff ssh.ds.xml + +module SSH; + +redef LogDataSeries::dump_schema = T; + +# Haven't yet found a way to check for the effect of these. +redef LogDataSeries::compression = "bz2"; +redef LogDataSeries::extent_size = 1000; +redef LogDataSeries::num_threads = 5; + +# LogDataSeries::use_integer_for_time is tested separately. + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro new file mode 100644 index 0000000000..7b708473e3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro @@ -0,0 +1,34 @@ +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries +# +# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out +# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt --skip-index $i; done >>out +# @TEST-EXEC: btest-diff out + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { LOG }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + } &log; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor_cmd = "echo"; + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); +} + +event new_connection(c: connection) + { + Log::write(Test::LOG, [$t=network_time(), $id=c$id]); + } diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro new file mode 100644 index 0000000000..ee0426ae55 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro @@ -0,0 +1,35 @@ +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries +# +# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES +# @TEST-EXEC: ds2txt --skip-index ssh.ds >ssh.ds.txt +# @TEST-EXEC: btest-diff ssh.ds.txt + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro new file mode 100644 index 0000000000..5e3f864b33 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro @@ -0,0 +1,9 @@ +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries +# +# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES +# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt +# @TEST-EXEC: btest-diff conn.ds.txt + +redef LogDataSeries::use_integer_for_time = T; diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro new file mode 100644 index 0000000000..ee1342c470 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro @@ -0,0 +1,9 @@ +# +# @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries +# +# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES +# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt +# @TEST-EXEC: ds2txt --skip-index http.ds >http.ds.txt +# @TEST-EXEC: btest-diff conn.ds.txt +# @TEST-EXEC: btest-diff http.ds.txt diff --git a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro index 7c06ff9248..3f6d40adaf 100644 --- a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro +++ b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT | egrep "test|test2" | sort >out +#@TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out # @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out # @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff .stderr diff --git a/testing/btest/scripts/base/frameworks/logging/rotate.bro b/testing/btest/scripts/base/frameworks/logging/rotate.bro index 14123c56c6..86f659c193 100644 --- a/testing/btest/scripts/base/frameworks/logging/rotate.bro +++ b/testing/btest/scripts/base/frameworks/logging/rotate.bro @@ -1,6 +1,6 @@ # -# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT 2>&1 | grep "test" >out -# @TEST-EXEC: for i in test.*.log; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 | grep "test" >out +# @TEST-EXEC: for i in `ls test.*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out # @TEST-EXEC: btest-diff out module Test; diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index c4e74f99fa..fba89fb724 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -10,7 +10,7 @@ BROPATH=`bash -c %(testbase)s/../../../build/bro-path-dev`:%(testbase)s/../scrip BRO_SEED_FILE=%(testbase)s/../random.seed TZ=UTC LC_ALL=C -PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(default_path)s +PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(testbase)s/../../scripts:%(default_path)s TEST_DIFF_CANONIFIER=%(testbase)s/../../scripts/diff-canonifier-external TEST_DIFF_BRIEF=1 TRACES=%(testbase)s/Traces diff --git a/testing/scripts/has-writer b/testing/scripts/has-writer new file mode 100755 index 0000000000..683d31041f --- /dev/null +++ b/testing/scripts/has-writer @@ -0,0 +1,6 @@ +#! /usr/bin/env bash +# +# Returns true if Bro has been compiled with support for writer type +# $1. The type name must match what "bro --help" prints. + +bro --helper 2>&1 | grep -qi "Supported log formats:.*$1"