mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Fix for when not producing local output; that hung.
* origin/topic/robin/dataseries: Moving trace for rotation test into traces directory. Fixing a rotation race condition at termination. Portability fixes. Extending DS docs with some examples. Updating doc. Fixing pack_scale and time-as-int. Adding format specifier to DS spec to print out double as %.6f. DataSeries updates and fixes. DataSeries tuning. Tweaking DataSeries support. Extending log post-processor call to include the name of the writer. Removing an unnecessary const cast. DataSeries TODO list with open issues/questions. Starting DataSeries HowTo. Additional test output canonification for ds2txt's timestamps. In threads, an internal error now immediately aborts. DataSeries cleanup. Working on DataSeries support. Merging in DataSeries support from topic/gilbert/logging. Fixing threads' DoFinish() method.
This commit is contained in:
Robin Sommer
commit
7cc863c5fc
52 changed files with 1844 additions and 133 deletions
|
|
@ -107,6 +107,21 @@ if (GOOGLEPERFTOOLS_FOUND)
|
|||
endif ()
|
||||
endif ()
|
||||
|
||||
set(USE_DATASERIES false)
|
||||
find_package(Lintel)
|
||||
find_package(DataSeries)
|
||||
find_package(LibXML2)
|
||||
|
||||
if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
|
||||
set(USE_DATASERIES true)
|
||||
include_directories(BEFORE ${Lintel_INCLUDE_DIR})
|
||||
include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
|
||||
include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
|
||||
list(APPEND OPTLIBS ${Lintel_LIBRARIES})
|
||||
list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
|
||||
list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
|
||||
endif()
|
||||
|
||||
if (ENABLE_PERFTOOLS_DEBUG)
|
||||
# Just a no op to prevent CMake from complaining about manually-specified
|
||||
# ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
|
||||
|
|
@ -198,6 +213,7 @@ message(
|
|||
"\nGeoIP: ${USE_GEOIP}"
|
||||
"\nGoogle perftools: ${USE_PERFTOOLS}"
|
||||
"\n debugging: ${USE_PERFTOOLS_DEBUG}"
|
||||
"\nDataSeries: ${USE_DATASERIES}"
|
||||
"\n"
|
||||
"\n================================================================\n"
|
||||
)
|
||||
|
|
|
|||
7
NEWS
7
NEWS
|
|
@ -47,6 +47,13 @@ Bro 2.1
|
|||
joint set of events. The `icmp_conn` record got a new boolean field
|
||||
'v6' that indicates whether the ICMP message is v4 or v6.
|
||||
|
||||
- Log postprocessor scripts get an additional argument indicating the
|
||||
type of the log writer in use (e.g., "ascii").
|
||||
|
||||
- BroControl's make-archive-name scripts also receives the writer
|
||||
type, but as it's 2nd(!) argument. If you're using a custom version
|
||||
of that script, you need to adapt it. See the shipped version for
|
||||
details.
|
||||
|
||||
TODO: Extend.
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 519d2e21ee375833c89eb6f7dc95c1eac3de17ab
|
||||
Subproject commit ed933502b4d2518f94b6cfa7a5b371e53fda5c3d
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 76876ce0e7da4888c91b3aea024c5cfd36405310
|
||||
Subproject commit e0da8d0e284bbebbaef711c91c1b961580f225d2
|
||||
2
cmake
2
cmake
|
|
@ -1 +1 @@
|
|||
Subproject commit 49278736c1404cb8c077272b80312c947e68bf52
|
||||
Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b
|
||||
|
|
@ -114,6 +114,9 @@
|
|||
/* Analyze Mobile IPv6 traffic */
|
||||
#cmakedefine ENABLE_MOBILE_IPV6
|
||||
|
||||
/* Use the DataSeries writer. */
|
||||
#cmakedefine USE_DATASERIES
|
||||
|
||||
/* Version number of package */
|
||||
#define VERSION "@VERSION@"
|
||||
|
||||
|
|
|
|||
9
configure
vendored
9
configure
vendored
|
|
@ -56,6 +56,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
|
|||
--with-ruby-lib=PATH path to ruby library
|
||||
--with-ruby-inc=PATH path to ruby headers
|
||||
--with-swig=PATH path to SWIG executable
|
||||
--with-dataseries=PATH path to DataSeries and Lintel libraries
|
||||
--with-xml2=PATH path to libxml2 installation (for DataSeries)
|
||||
|
||||
Packaging Options (for developers):
|
||||
--binary-package toggle special logic for binary packaging
|
||||
|
|
@ -213,6 +215,13 @@ while [ $# -ne 0 ]; do
|
|||
--with-swig=*)
|
||||
append_cache_entry SWIG_EXECUTABLE PATH $optarg
|
||||
;;
|
||||
--with-dataseries=*)
|
||||
append_cache_entry DataSeries_ROOT_DIR PATH $optarg
|
||||
append_cache_entry Lintel_ROOT_DIR PATH $optarg
|
||||
;;
|
||||
--with-xml2=*)
|
||||
append_cache_entry LibXML2_ROOT_DIR PATH $optarg
|
||||
;;
|
||||
--binary-package)
|
||||
append_cache_entry BINARY_PACKAGING_MODE BOOL true
|
||||
;;
|
||||
|
|
|
|||
168
doc/logging-dataseries.rst
Normal file
168
doc/logging-dataseries.rst
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
|
||||
=============================
|
||||
Binary Output with DataSeries
|
||||
=============================
|
||||
|
||||
.. rst-class:: opening
|
||||
|
||||
Bro's default ASCII log format is not exactly the most efficient
|
||||
way for storing large volumes of data. An an alternative, Bro comes
|
||||
with experimental support for `DataSeries
|
||||
<http://www.hpl.hp.com/techreports/2009/HPL-2009-323.html>`_
|
||||
output, an efficient binary format for recording structured bulk
|
||||
data. DataSeries is developed and maintained at HP Labs.
|
||||
|
||||
.. contents::
|
||||
|
||||
Installing DataSeries
|
||||
---------------------
|
||||
|
||||
To use DataSeries, its libraries must be available at compile-time,
|
||||
along with the supporting *Lintel* package. Generally, both are
|
||||
distributed on `HP Labs' web site
|
||||
<http://tesla.hpl.hp.com/opensource/>`_. Currently, however, you need
|
||||
to use recent developments versions for both packages, which you can
|
||||
download from github like this::
|
||||
|
||||
git clone http://github.com/dataseries/Lintel
|
||||
git clone http://github.com/dataseries/DataSeries
|
||||
|
||||
To build and install the two into ``<prefix>``, do::
|
||||
|
||||
( cd Lintel && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
|
||||
( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
|
||||
|
||||
Please refer to the packages' documentation for more information about
|
||||
the installation process. In particular, there's more information on
|
||||
required and optional `dependencies for Lintel
|
||||
<https://raw.github.com/eric-anderson/Lintel/master/doc/dependencies.txt>`_
|
||||
and `dependencies for DataSeries
|
||||
<https://raw.github.com/eric-anderson/DataSeries/master/doc/dependencies.txt>`_
|
||||
|
||||
Compiling Bro with DataSeries Support
|
||||
-------------------------------------
|
||||
|
||||
Once you have installed DataSeries, Bro's ``configure`` should pick it
|
||||
up automatically as long as it finds it in a standard system location.
|
||||
Alternatively, you can specify the DataSeries installation prefix
|
||||
manually with ``--with-dataseries=<prefix>``. Keep an eye on
|
||||
``configure``'s summary output, if it looks like the following, Bro
|
||||
found DataSeries and will compile in the support::
|
||||
|
||||
# ./configure --with-dataseries=/usr/local
|
||||
[...]
|
||||
====================| Bro Build Summary |=====================
|
||||
[...]
|
||||
DataSeries: true
|
||||
[...]
|
||||
================================================================
|
||||
|
||||
Activating DataSeries
|
||||
---------------------
|
||||
|
||||
The direct way to use DataSeries is to switch *all* log files over to
|
||||
the binary format. To do that, just add ``redef
|
||||
Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``.
|
||||
For testing, you can also just pass that on the command line::
|
||||
|
||||
bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES
|
||||
|
||||
With that, Bro will now write all its output into DataSeries files
|
||||
``*.ds``. You can inspect these using DataSeries's set of command line
|
||||
tools, which its installation process installs into ``<prefix>/bin``.
|
||||
For example, to convert a file back into an ASCII representation::
|
||||
|
||||
$ ds2txt conn.log
|
||||
[... We skip a bunch of meta data here ...]
|
||||
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||
1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
|
||||
1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
|
||||
1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
|
||||
1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117
|
||||
1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127
|
||||
1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211
|
||||
[...]
|
||||
|
||||
(``--skip-all`` suppresses the meta data.)
|
||||
|
||||
Note that the ASCII conversion is *not* equivalent to Bro's default
|
||||
output format.
|
||||
|
||||
You can also switch only individual files over to DataSeries by adding
|
||||
code like this to your ``local.bro``::
|
||||
|
||||
.. code:: bro
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
|
||||
f$writer = Log::WRITER_DATASERIES; # Change writer type.
|
||||
Log::add_filter(Conn::LOG, f); # Replace filter with adapted version.
|
||||
}
|
||||
|
||||
Bro's DataSeries writer comes with a few tuning options, see
|
||||
:doc:`scripts/base/frameworks/logging/writers/dataseries`.
|
||||
|
||||
Working with DataSeries
|
||||
=======================
|
||||
|
||||
Here are few examples of using DataSeries command line tools to work
|
||||
with the output files.
|
||||
|
||||
* Printing CSV::
|
||||
|
||||
$ ds2txt --csv conn.log
|
||||
ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes
|
||||
1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0
|
||||
1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0
|
||||
1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0
|
||||
1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0
|
||||
[...]
|
||||
|
||||
Add ``--separator=X`` to set a different separator.
|
||||
|
||||
* Extracting a subset of columns::
|
||||
|
||||
$ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log
|
||||
1258790493.773208 192.168.1.255 137
|
||||
1258790451.402091 192.168.1.255 138
|
||||
1258790493.787448 192.168.1.255 138
|
||||
1258790615.268111 192.168.1.255 137
|
||||
1258790615.289842 192.168.1.255 138
|
||||
[...]
|
||||
|
||||
* Filtering rows::
|
||||
|
||||
$ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all conn.ds
|
||||
1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp 15.004568 798 0 S0 F 0 D 6 966 0 0
|
||||
1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp 15.004581 798 0 S0 F 0 D 6 966 0 0
|
||||
1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp 15.005071 798 0 S0 F 0 D 6 966 0 0
|
||||
1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp 15.004824 798 0 S0 F 0 D 6 966 0 0
|
||||
1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp 15.005078 798 0 S0 F 0 D 6 966 0 0
|
||||
1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp 15.004082 798 0 S0 F 0 D 6 966 0 0
|
||||
[...]
|
||||
|
||||
* Calculate some statistics:
|
||||
|
||||
Mean/stdev/min/max over a column::
|
||||
|
||||
$ dsstatgroupby '*' basic duration from conn.ds
|
||||
# Begin DSStatGroupByModule
|
||||
# processed 2159 rows, where clause eliminated 0 rows
|
||||
# count(*), mean(duration), stddev, min, max
|
||||
2159, 42.7938, 1858.34, 0, 86370
|
||||
[...]
|
||||
|
||||
Quantiles of total connection volume::
|
||||
|
||||
> dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds
|
||||
[...]
|
||||
2159 data points, mean 24616 +- 343295 [0,1.26615e+07]
|
||||
quantiles about every 216 data points:
|
||||
10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469
|
||||
tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262
|
||||
[...]
|
||||
|
||||
The ``man`` pages for these tool show further options, and their
|
||||
``-h`` option gives some more information (either can be a bit cryptic
|
||||
unfortunately though).
|
||||
|
|
@ -36,6 +36,7 @@ rest_target(${psd} base/frameworks/logging/main.bro)
|
|||
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
|
||||
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/main.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
@load ./main
|
||||
@load ./postprocessors
|
||||
@load ./writers/ascii
|
||||
@load ./writers/dataseries
|
||||
|
|
|
|||
|
|
@ -332,7 +332,7 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
|
|||
function default_path_func(id: ID, path: string, rec: any) : string
|
||||
{
|
||||
local id_str = fmt("%s", id);
|
||||
|
||||
|
||||
local parts = split1(id_str, /::/);
|
||||
if ( |parts| == 2 )
|
||||
{
|
||||
|
|
@ -340,7 +340,7 @@ function default_path_func(id: ID, path: string, rec: any) : string
|
|||
# or a filter path explicitly set by the user, so continue using it.
|
||||
if ( path != "" )
|
||||
return path;
|
||||
|
||||
|
||||
# Example: Notice::LOG -> "notice"
|
||||
if ( parts[2] == "LOG" )
|
||||
{
|
||||
|
|
@ -356,11 +356,11 @@ function default_path_func(id: ID, path: string, rec: any) : string
|
|||
output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|));
|
||||
return to_lower(output);
|
||||
}
|
||||
|
||||
|
||||
# Example: Notice::POLICY_LOG -> "notice_policy"
|
||||
if ( /_LOG$/ in parts[2] )
|
||||
parts[2] = sub(parts[2], /_LOG$/, "");
|
||||
|
||||
|
||||
return cat(to_lower(parts[1]),"_",to_lower(parts[2]));
|
||||
}
|
||||
else
|
||||
|
|
@ -376,13 +376,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo
|
|||
if ( pp_cmd == "" )
|
||||
return T;
|
||||
|
||||
# Turn, e.g., Log::WRITER_ASCII into "ascii".
|
||||
local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", "");
|
||||
|
||||
# The date format is hard-coded here to provide a standardized
|
||||
# script interface.
|
||||
system(fmt("%s %s %s %s %s %d",
|
||||
system(fmt("%s %s %s %s %s %d %s",
|
||||
pp_cmd, npath, info$path,
|
||||
strftime("%y-%m-%d_%H.%M.%S", info$open),
|
||||
strftime("%y-%m-%d_%H.%M.%S", info$close),
|
||||
info$terminating));
|
||||
info$terminating, writer));
|
||||
|
||||
return T;
|
||||
}
|
||||
|
|
@ -407,7 +410,7 @@ function add_filter(id: ID, filter: Filter) : bool
|
|||
# definition.
|
||||
if ( ! filter?$path_func )
|
||||
filter$path_func = default_path_func;
|
||||
|
||||
|
||||
filters[id, filter$name] = filter;
|
||||
return __add_filter(id, filter);
|
||||
}
|
||||
|
|
|
|||
60
scripts/base/frameworks/logging/writers/dataseries.bro
Normal file
60
scripts/base/frameworks/logging/writers/dataseries.bro
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
##! Interface for the DataSeries log writer.
|
||||
|
||||
module LogDataSeries;
|
||||
|
||||
export {
|
||||
## Compression to use with the DS output file. Options are:
|
||||
##
|
||||
## 'none' -- No compression.
|
||||
## 'lzf' -- LZF compression. Very quick, but leads to larger output files.
|
||||
## 'lzo' -- LZO compression. Very fast decompression times.
|
||||
## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output.
|
||||
## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
|
||||
const compression = "lzo" &redef;
|
||||
|
||||
## The extent buffer size.
|
||||
## Larger values here lead to better compression and more efficient writes, but
|
||||
## also increase the lag between the time events are received and the time they
|
||||
## are actually written to disk.
|
||||
const extent_size = 65536 &redef;
|
||||
|
||||
## Should we dump the XML schema we use for this DS file to disk?
|
||||
## If yes, the XML schema shares the name of the logfile, but has
|
||||
## an XML ending.
|
||||
const dump_schema = F &redef;
|
||||
|
||||
## How many threads should DataSeries spawn to perform compression?
|
||||
## Note that this dictates the number of threads per log stream. If
|
||||
## you're using a lot of streams, you may want to keep this number
|
||||
## relatively small.
|
||||
##
|
||||
## Default value is 1, which will spawn one thread / stream.
|
||||
##
|
||||
## Maximum is 128, minimum is 1.
|
||||
const num_threads = 1 &redef;
|
||||
|
||||
## Should time be stored as an integer or a double?
|
||||
## Storing time as a double leads to possible precision issues and
|
||||
## can (significantly) increase the size of the resulting DS log.
|
||||
## That said, timestamps stored in double form are consistent
|
||||
## with the rest of Bro, including the standard ASCII log. Hence, we
|
||||
## use them by default.
|
||||
const use_integer_for_time = F &redef;
|
||||
}
|
||||
|
||||
# Default function to postprocess a rotated DataSeries log file. It moves the
|
||||
# rotated file to a new name that includes a timestamp with the opening time, and
|
||||
# then runs the writer's default postprocessor command on it.
|
||||
function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
|
||||
{
|
||||
# Move file to name including both opening and closing time.
|
||||
local dst = fmt("%s.%s.ds", info$path,
|
||||
strftime(Log::default_rotation_date_format, info$open));
|
||||
|
||||
system(fmt("/bin/mv %s %s", info$fname, dst));
|
||||
|
||||
# Run default postprocessor.
|
||||
return Log::run_rotation_postprocessor_cmd(info, dst);
|
||||
}
|
||||
|
||||
redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };
|
||||
|
|
@ -417,6 +417,7 @@ set(bro_SRCS
|
|||
logging/WriterBackend.cc
|
||||
logging/WriterFrontend.cc
|
||||
logging/writers/Ascii.cc
|
||||
logging/writers/DataSeries.cc
|
||||
logging/writers/None.cc
|
||||
|
||||
nb_dns.c
|
||||
|
|
|
|||
|
|
@ -15,10 +15,9 @@
|
|||
|
||||
extern int generate_documentation;
|
||||
|
||||
// Note: This function must be thread-safe.
|
||||
const char* type_name(TypeTag t)
|
||||
{
|
||||
static char errbuf[512];
|
||||
|
||||
static const char* type_names[int(NUM_TYPES)] = {
|
||||
"void",
|
||||
"bool", "int", "count", "counter",
|
||||
|
|
@ -37,10 +36,7 @@ const char* type_name(TypeTag t)
|
|||
};
|
||||
|
||||
if ( int(t) >= NUM_TYPES )
|
||||
{
|
||||
snprintf(errbuf, sizeof(errbuf), "%d: not a type tag", int(t));
|
||||
return errbuf;
|
||||
}
|
||||
return "type_name(): not a type tag";
|
||||
|
||||
return type_names[int(t)];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,3 +72,12 @@ const set_separator: string;
|
|||
const empty_field: string;
|
||||
const unset_field: string;
|
||||
|
||||
# Options for the DataSeries writer.
|
||||
|
||||
module LogDataSeries;
|
||||
|
||||
const compression: string;
|
||||
const extent_size: count;
|
||||
const dump_schema: bool;
|
||||
const use_integer_for_time: bool;
|
||||
const num_threads: count;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include "../NetVar.h"
|
||||
#include "../Net.h"
|
||||
|
||||
#include "threading/Manager.h"
|
||||
#include "threading/SerialTypes.h"
|
||||
|
||||
#include "Manager.h"
|
||||
|
|
@ -16,9 +17,11 @@
|
|||
#include "writers/Ascii.h"
|
||||
#include "writers/None.h"
|
||||
|
||||
#ifdef USE_DATASERIES
|
||||
#include "writers/DataSeries.h"
|
||||
#endif
|
||||
|
||||
using namespace logging;
|
||||
using threading::Value;
|
||||
using threading::Field;
|
||||
|
||||
// Structure describing a log writer type.
|
||||
struct WriterDefinition {
|
||||
|
|
@ -32,6 +35,9 @@ struct WriterDefinition {
|
|||
WriterDefinition log_writers[] = {
|
||||
{ BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate },
|
||||
{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
|
||||
#ifdef USE_DATASERIES
|
||||
{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
|
||||
#endif
|
||||
|
||||
// End marker, don't touch.
|
||||
{ BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 }
|
||||
|
|
@ -51,7 +57,7 @@ struct Manager::Filter {
|
|||
Func* postprocessor;
|
||||
|
||||
int num_fields;
|
||||
Field** fields;
|
||||
threading::Field** fields;
|
||||
|
||||
// Vector indexed by field number. Each element is a list of record
|
||||
// indices defining a path leading to the value across potential
|
||||
|
|
@ -119,6 +125,7 @@ Manager::Stream::~Stream()
|
|||
|
||||
Manager::Manager()
|
||||
{
|
||||
rotations_pending = 0;
|
||||
}
|
||||
|
||||
Manager::~Manager()
|
||||
|
|
@ -127,6 +134,16 @@ Manager::~Manager()
|
|||
delete *s;
|
||||
}
|
||||
|
||||
list<string> Manager::SupportedFormats()
|
||||
{
|
||||
list<string> formats;
|
||||
|
||||
for ( WriterDefinition* ld = log_writers; ld->type != BifEnum::Log::WRITER_DEFAULT; ++ld )
|
||||
formats.push_back(ld->name);
|
||||
|
||||
return formats;
|
||||
}
|
||||
|
||||
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
||||
{
|
||||
WriterDefinition* ld = log_writers;
|
||||
|
|
@ -135,7 +152,7 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
|||
{
|
||||
if ( ld->type == BifEnum::Log::WRITER_DEFAULT )
|
||||
{
|
||||
reporter->Error("unknow writer when creating writer");
|
||||
reporter->Error("unknown writer type requested");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -159,10 +176,8 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
|||
// function.
|
||||
ld->factory = 0;
|
||||
|
||||
DBG_LOG(DBG_LOGGING, "failed to init writer class %s",
|
||||
ld->name);
|
||||
|
||||
return false;
|
||||
reporter->Error("initialization of writer %s failed", ld->name);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -449,7 +464,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
|
|||
|
||||
filter->indices.push_back(new_indices);
|
||||
|
||||
filter->fields = (Field**)
|
||||
filter->fields = (threading::Field**)
|
||||
realloc(filter->fields,
|
||||
sizeof(Field) * ++filter->num_fields);
|
||||
|
||||
|
|
@ -459,7 +474,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
|
|||
return false;
|
||||
}
|
||||
|
||||
Field* field = new Field();
|
||||
threading::Field* field = new threading::Field();
|
||||
field->name = new_path;
|
||||
field->type = t->Tag();
|
||||
|
||||
|
|
@ -571,7 +586,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
|
|||
|
||||
for ( int i = 0; i < filter->num_fields; i++ )
|
||||
{
|
||||
Field* field = filter->fields[i];
|
||||
threading::Field* field = filter->fields[i];
|
||||
DBG_LOG(DBG_LOGGING, " field %10s: %s",
|
||||
field->name.c_str(), type_name(field->type));
|
||||
}
|
||||
|
|
@ -743,10 +758,10 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
|
|||
|
||||
// Copy the fields for WriterFrontend::Init() as it
|
||||
// will take ownership.
|
||||
Field** arg_fields = new Field*[filter->num_fields];
|
||||
threading::Field** arg_fields = new threading::Field*[filter->num_fields];
|
||||
|
||||
for ( int j = 0; j < filter->num_fields; ++j )
|
||||
arg_fields[j] = new Field(*filter->fields[j]);
|
||||
arg_fields[j] = new threading::Field(*filter->fields[j]);
|
||||
|
||||
writer = CreateWriter(stream->id, filter->writer,
|
||||
path, filter->num_fields,
|
||||
|
|
@ -897,10 +912,10 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
|
|||
return lval;
|
||||
}
|
||||
|
||||
Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
||||
threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
||||
RecordVal* columns)
|
||||
{
|
||||
Value** vals = new Value*[filter->num_fields];
|
||||
threading::Value** vals = new threading::Value*[filter->num_fields];
|
||||
|
||||
for ( int i = 0; i < filter->num_fields; ++i )
|
||||
{
|
||||
|
|
@ -919,7 +934,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
|||
if ( ! val )
|
||||
{
|
||||
// Value, or any of its parents, is not set.
|
||||
vals[i] = new Value(filter->fields[i]->type, false);
|
||||
vals[i] = new threading::Value(filter->fields[i]->type, false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -932,7 +947,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
|||
}
|
||||
|
||||
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
|
||||
int num_fields, const Field* const* fields, bool local, bool remote)
|
||||
int num_fields, const threading::Field* const* fields, bool local, bool remote)
|
||||
{
|
||||
Stream* stream = FindStream(id);
|
||||
|
||||
|
|
@ -996,7 +1011,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
|
|||
return writer_obj;
|
||||
}
|
||||
|
||||
void Manager::DeleteVals(int num_fields, Value** vals)
|
||||
void Manager::DeleteVals(int num_fields, threading::Value** vals)
|
||||
{
|
||||
// Note this code is duplicated in WriterBackend::DeleteVals().
|
||||
for ( int i = 0; i < num_fields; i++ )
|
||||
|
|
@ -1006,7 +1021,7 @@ void Manager::DeleteVals(int num_fields, Value** vals)
|
|||
}
|
||||
|
||||
bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields,
|
||||
Value** vals)
|
||||
threading::Value** vals)
|
||||
{
|
||||
Stream* stream = FindStream(id);
|
||||
|
||||
|
|
@ -1113,10 +1128,19 @@ bool Manager::Flush(EnumVal* id)
|
|||
|
||||
void Manager::Terminate()
|
||||
{
|
||||
// Make sure we process all the pending rotations.
|
||||
while ( rotations_pending )
|
||||
{
|
||||
thread_mgr->ForceProcessing(); // A blatant layering violation ...
|
||||
usleep(1000);
|
||||
}
|
||||
|
||||
for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
|
||||
{
|
||||
if ( *s )
|
||||
Flush((*s)->id);
|
||||
if ( ! *s )
|
||||
continue;
|
||||
|
||||
Flush((*s)->id);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1219,11 +1243,19 @@ void Manager::Rotate(WriterInfo* winfo)
|
|||
|
||||
// Trigger the rotation.
|
||||
winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
|
||||
|
||||
++rotations_pending;
|
||||
}
|
||||
|
||||
bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
|
||||
double open, double close, bool terminating)
|
||||
{
|
||||
--rotations_pending;
|
||||
|
||||
if ( ! writer )
|
||||
// Writer didn't produce local output.
|
||||
return true;
|
||||
|
||||
DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
|
||||
writer->Path().c_str(), network_time, new_name.c_str());
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ class RotationTimer;
|
|||
|
||||
namespace logging {
|
||||
|
||||
|
||||
class WriterBackend;
|
||||
class WriterFrontend;
|
||||
class RotationFinishedMessage;
|
||||
|
|
@ -56,7 +55,7 @@ public:
|
|||
* logging.bif, which just forwards here.
|
||||
*/
|
||||
bool EnableStream(EnumVal* id);
|
||||
|
||||
|
||||
/**
|
||||
* Disables a log stream.
|
||||
*
|
||||
|
|
@ -145,6 +144,11 @@ public:
|
|||
*/
|
||||
void Terminate();
|
||||
|
||||
/**
|
||||
* Returns a list of supported output formats.
|
||||
*/
|
||||
static list<string> SupportedFormats();
|
||||
|
||||
protected:
|
||||
friend class WriterFrontend;
|
||||
friend class RotationFinishedMessage;
|
||||
|
|
@ -196,6 +200,7 @@ private:
|
|||
WriterInfo* FindWriter(WriterFrontend* writer);
|
||||
|
||||
vector<Stream *> streams; // Indexed by stream enum.
|
||||
int rotations_pending; // Number of rotations not yet finished.
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -223,17 +223,6 @@ bool WriterBackend::Flush()
|
|||
return true;
|
||||
}
|
||||
|
||||
bool WriterBackend::Finish()
|
||||
{
|
||||
if ( ! DoFlush() )
|
||||
{
|
||||
DisableFrontend();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WriterBackend::DoHeartbeat(double network_time, double current_time)
|
||||
{
|
||||
MsgThread::DoHeartbeat(network_time, current_time);
|
||||
|
|
@ -279,4 +268,9 @@ string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
|
|||
return s;
|
||||
}
|
||||
|
||||
|
||||
string WriterBackend::Render(double d) const
|
||||
{
|
||||
char buf[256];
|
||||
modp_dtoa(d, buf, 6);
|
||||
return buf;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,15 +101,6 @@ public:
|
|||
*/
|
||||
bool Rotate(string rotated_path, double open, double close, bool terminating);
|
||||
|
||||
/**
|
||||
* Finishes writing to this logger in a regularl fashion. Must not be
|
||||
* called if an error has been indicated earlier. After calling this,
|
||||
* no further writing must be performed.
|
||||
*
|
||||
* @return False if an error occured.
|
||||
*/
|
||||
bool Finish();
|
||||
|
||||
/**
|
||||
* Disables the frontend that has instantiated this backend. Once
|
||||
* disabled,the frontend will not send any further message over.
|
||||
|
|
@ -174,7 +165,17 @@ public:
|
|||
*/
|
||||
string Render(const threading::Value::subnet_t& subnet) const;
|
||||
|
||||
/** Helper method to render a double in Bro's standard precision.
|
||||
*
|
||||
* @param d The double.
|
||||
*
|
||||
* @return An ASCII representation of the double.
|
||||
*/
|
||||
string Render(double d) const;
|
||||
|
||||
protected:
|
||||
friend class FinishMessage;
|
||||
|
||||
/**
|
||||
* Writer-specific intialization method.
|
||||
*
|
||||
|
|
@ -272,26 +273,18 @@ protected:
|
|||
bool terminating) = 0;
|
||||
|
||||
/**
|
||||
* Writer-specific method implementing log output finalization at
|
||||
* termination. Not called when any of the other methods has
|
||||
* previously signaled an error, i.e., executing this method signals
|
||||
* a regular shutdown of the writer.
|
||||
* Writer-specific method called just before the threading system is
|
||||
* going to shutdown.
|
||||
*
|
||||
* A writer implementation must override this method but it can just
|
||||
* ignore calls if flushing doesn't align with its semantics.
|
||||
*
|
||||
* If the method returns false, it will be assumed that a fatal error
|
||||
* has occured that prevents the writer from further operation; it
|
||||
* will then be disabled and eventually deleted. When returning
|
||||
* false, an implementation should also call Error() to indicate what
|
||||
* happened.
|
||||
* This method can be overridden but one must call
|
||||
* WriterBackend::DoFinish().
|
||||
*/
|
||||
virtual bool DoFinish() = 0;
|
||||
virtual bool DoFinish() { return MsgThread::DoFinish(); }
|
||||
|
||||
/**
|
||||
* Triggered by regular heartbeat messages from the main thread.
|
||||
*
|
||||
* This method can be overridden but once must call
|
||||
* This method can be overridden but one must call
|
||||
* WriterBackend::DoHeartbeat().
|
||||
*/
|
||||
virtual bool DoHeartbeat(double network_time, double current_time);
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ public:
|
|||
FinishMessage(WriterBackend* backend)
|
||||
: threading::InputMessage<WriterBackend>("Finish", backend) {}
|
||||
|
||||
virtual bool Process() { return Object()->Finish(); }
|
||||
virtual bool Process() { return Object()->DoFinish(); }
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -117,8 +117,9 @@ WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool ar
|
|||
if ( local )
|
||||
{
|
||||
backend = log_mgr->CreateBackend(this, writer->AsEnum());
|
||||
assert(backend);
|
||||
backend->Start();
|
||||
|
||||
if ( backend )
|
||||
backend->Start();
|
||||
}
|
||||
|
||||
else
|
||||
|
|
@ -256,6 +257,10 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool
|
|||
|
||||
if ( backend )
|
||||
backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
|
||||
else
|
||||
// Still signal log manager that we're done, but signal that
|
||||
// nothing happened by setting the writer to zeri.
|
||||
log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating);
|
||||
}
|
||||
|
||||
void WriterFrontend::Finish()
|
||||
|
|
|
|||
|
|
@ -69,8 +69,7 @@ bool Ascii::WriteHeaderField(const string& key, const string& val)
|
|||
return (fwrite(str.c_str(), str.length(), 1, file) == 1);
|
||||
}
|
||||
|
||||
bool Ascii::DoInit(string path, int num_fields,
|
||||
const Field* const * fields)
|
||||
bool Ascii::DoInit(string path, int num_fields, const Field* const * fields)
|
||||
{
|
||||
if ( output_to_stdout )
|
||||
path = "/dev/stdout";
|
||||
|
|
@ -87,6 +86,9 @@ bool Ascii::DoInit(string path, int num_fields,
|
|||
|
||||
if ( include_header )
|
||||
{
|
||||
string names;
|
||||
string types;
|
||||
|
||||
string str = string(header_prefix, header_prefix_len)
|
||||
+ "separator " // Always use space as separator here.
|
||||
+ get_escaped_string(string(separator, separator_len), false)
|
||||
|
|
@ -104,9 +106,6 @@ bool Ascii::DoInit(string path, int num_fields,
|
|||
WriteHeaderField("path", get_escaped_string(path, false))) )
|
||||
goto write_error;
|
||||
|
||||
string names;
|
||||
string types;
|
||||
|
||||
for ( int i = 0; i < num_fields; ++i )
|
||||
{
|
||||
if ( i > 0 )
|
||||
|
|
@ -115,15 +114,8 @@ bool Ascii::DoInit(string path, int num_fields,
|
|||
types += string(separator, separator_len);
|
||||
}
|
||||
|
||||
const Field* field = fields[i];
|
||||
names += field->name;
|
||||
types += type_name(field->type);
|
||||
if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) )
|
||||
{
|
||||
types += "[";
|
||||
types += type_name(field->subtype);
|
||||
types += "]";
|
||||
}
|
||||
names += fields[i]->name;
|
||||
types += fields[i]->TypeName();
|
||||
}
|
||||
|
||||
if ( ! (WriteHeaderField("fields", names)
|
||||
|
|
@ -146,7 +138,7 @@ bool Ascii::DoFlush()
|
|||
|
||||
bool Ascii::DoFinish()
|
||||
{
|
||||
return true;
|
||||
return WriterBackend::DoFinish();
|
||||
}
|
||||
|
||||
bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
|
||||
|
|
@ -184,15 +176,19 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
|
|||
desc->Add(Render(val->val.addr_val));
|
||||
break;
|
||||
|
||||
case TYPE_TIME:
|
||||
case TYPE_INTERVAL:
|
||||
char buf[256];
|
||||
modp_dtoa(val->val.double_val, buf, 6);
|
||||
desc->Add(buf);
|
||||
case TYPE_DOUBLE:
|
||||
// Rendering via Add() truncates trailing 0s after the
|
||||
// decimal point. The difference with TIME/INTERVAL is mainly
|
||||
// to keep the log format consistent.
|
||||
desc->Add(val->val.double_val);
|
||||
break;
|
||||
|
||||
case TYPE_DOUBLE:
|
||||
desc->Add(val->val.double_val);
|
||||
case TYPE_INTERVAL:
|
||||
case TYPE_TIME:
|
||||
// Rendering via Render() keeps trailing 0s after the decimal
|
||||
// point. The difference with DOUBLEis mainly to keep the log
|
||||
// format consistent.
|
||||
desc->Add(Render(val->val.double_val));
|
||||
break;
|
||||
|
||||
case TYPE_ENUM:
|
||||
|
|
|
|||
417
src/logging/writers/DataSeries.cc
Normal file
417
src/logging/writers/DataSeries.cc
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
// See the file "COPYING" in the main distribution directory for copyright.
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <errno.h>
|
||||
|
||||
#include <DataSeries/GeneralField.hpp>
|
||||
|
||||
#include "NetVar.h"
|
||||
#include "threading/SerialTypes.h"
|
||||
|
||||
#include "DataSeries.h"
|
||||
|
||||
using namespace logging;
|
||||
using namespace writer;
|
||||
|
||||
std::string DataSeries::LogValueToString(threading::Value *val)
|
||||
{
|
||||
// In some cases, no value is attached. If this is the case, return
|
||||
// an empty string.
|
||||
if( ! val->present )
|
||||
return "";
|
||||
|
||||
switch(val->type) {
|
||||
case TYPE_BOOL:
|
||||
return (val->val.int_val ? "true" : "false");
|
||||
|
||||
case TYPE_INT:
|
||||
{
|
||||
std::ostringstream ostr;
|
||||
ostr << val->val.int_val;
|
||||
return ostr.str();
|
||||
}
|
||||
|
||||
case TYPE_COUNT:
|
||||
case TYPE_COUNTER:
|
||||
case TYPE_PORT:
|
||||
{
|
||||
std::ostringstream ostr;
|
||||
ostr << val->val.uint_val;
|
||||
return ostr.str();
|
||||
}
|
||||
|
||||
case TYPE_SUBNET:
|
||||
return Render(val->val.subnet_val);
|
||||
|
||||
case TYPE_ADDR:
|
||||
return Render(val->val.addr_val);
|
||||
|
||||
// Note: These two cases are relatively special. We need to convert
|
||||
// these values into their integer equivalents to maximize precision.
|
||||
// At the moment, there won't be a noticeable effect (Bro uses the
|
||||
// double format everywhere internally, so we've already lost the
|
||||
// precision we'd gain here), but timestamps may eventually switch to
|
||||
// this representation within Bro.
|
||||
//
|
||||
// In the near-term, this *should* lead to better pack_relative (and
|
||||
// thus smaller output files).
|
||||
case TYPE_TIME:
|
||||
case TYPE_INTERVAL:
|
||||
if ( ds_use_integer_for_time )
|
||||
{
|
||||
std::ostringstream ostr;
|
||||
ostr << (uint64_t)(DataSeries::TIME_SCALE * val->val.double_val);
|
||||
return ostr.str();
|
||||
}
|
||||
else
|
||||
return Render(val->val.double_val);
|
||||
|
||||
case TYPE_DOUBLE:
|
||||
return Render(val->val.double_val);
|
||||
|
||||
case TYPE_ENUM:
|
||||
case TYPE_STRING:
|
||||
case TYPE_FILE:
|
||||
case TYPE_FUNC:
|
||||
if ( ! val->val.string_val->size() )
|
||||
return "";
|
||||
|
||||
return string(val->val.string_val->data(), val->val.string_val->size());
|
||||
|
||||
case TYPE_TABLE:
|
||||
{
|
||||
if ( ! val->val.set_val.size )
|
||||
return "";
|
||||
|
||||
string tmpString = "";
|
||||
|
||||
for ( int j = 0; j < val->val.set_val.size; j++ )
|
||||
{
|
||||
if ( j > 0 )
|
||||
tmpString += ds_set_separator;
|
||||
|
||||
tmpString += LogValueToString(val->val.set_val.vals[j]);
|
||||
}
|
||||
|
||||
return tmpString;
|
||||
}
|
||||
|
||||
case TYPE_VECTOR:
|
||||
{
|
||||
if ( ! val->val.vector_val.size )
|
||||
return "";
|
||||
|
||||
string tmpString = "";
|
||||
|
||||
for ( int j = 0; j < val->val.vector_val.size; j++ )
|
||||
{
|
||||
if ( j > 0 )
|
||||
tmpString += ds_set_separator;
|
||||
|
||||
tmpString += LogValueToString(val->val.vector_val.vals[j]);
|
||||
}
|
||||
|
||||
return tmpString;
|
||||
}
|
||||
|
||||
default:
|
||||
InternalError(Fmt("unknown type %s in DataSeries::LogValueToString", type_name(val->type)));
|
||||
return "cannot be reached";
|
||||
}
|
||||
}
|
||||
|
||||
string DataSeries::GetDSFieldType(const threading::Field *field)
|
||||
{
|
||||
switch(field->type) {
|
||||
case TYPE_BOOL:
|
||||
return "bool";
|
||||
|
||||
case TYPE_COUNT:
|
||||
case TYPE_COUNTER:
|
||||
case TYPE_PORT:
|
||||
case TYPE_INT:
|
||||
return "int64";
|
||||
|
||||
case TYPE_DOUBLE:
|
||||
return "double";
|
||||
|
||||
case TYPE_TIME:
|
||||
case TYPE_INTERVAL:
|
||||
return ds_use_integer_for_time ? "int64" : "double";
|
||||
|
||||
case TYPE_SUBNET:
|
||||
case TYPE_ADDR:
|
||||
case TYPE_ENUM:
|
||||
case TYPE_STRING:
|
||||
case TYPE_FILE:
|
||||
case TYPE_TABLE:
|
||||
case TYPE_VECTOR:
|
||||
case TYPE_FUNC:
|
||||
return "variable32";
|
||||
|
||||
default:
|
||||
InternalError(Fmt("unknown type %s in DataSeries::GetDSFieldType", type_name(field->type)));
|
||||
return "cannot be reached";
|
||||
}
|
||||
}
|
||||
|
||||
string DataSeries::BuildDSSchemaFromFieldTypes(const vector<SchemaValue>& vals, string sTitle)
|
||||
{
|
||||
if( ! sTitle.size() )
|
||||
sTitle = "GenericBroStream";
|
||||
|
||||
string xmlschema = "<ExtentType name=\""
|
||||
+ sTitle
|
||||
+ "\" version=\"1.0\" namespace=\"bro-ids.org\">\n";
|
||||
|
||||
for( size_t i = 0; i < vals.size(); ++i )
|
||||
{
|
||||
xmlschema += "\t<field type=\""
|
||||
+ vals[i].ds_type
|
||||
+ "\" name=\""
|
||||
+ vals[i].field_name
|
||||
+ "\" " + vals[i].field_options
|
||||
+ "/>\n";
|
||||
}
|
||||
|
||||
xmlschema += "</ExtentType>\n";
|
||||
|
||||
for( size_t i = 0; i < vals.size(); ++i )
|
||||
{
|
||||
xmlschema += "<!-- " + vals[i].field_name
|
||||
+ " : " + vals[i].bro_type
|
||||
+ " -->\n";
|
||||
}
|
||||
|
||||
return xmlschema;
|
||||
}
|
||||
|
||||
std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
|
||||
{
|
||||
switch( field->type ) {
|
||||
case TYPE_TIME:
|
||||
case TYPE_INTERVAL:
|
||||
{
|
||||
std::string s;
|
||||
s += "pack_relative=\"" + std::string(field->name) + "\"";
|
||||
|
||||
if ( ! ds_use_integer_for_time )
|
||||
s += " pack_scale=\"1e-6\" print_format=\"%.6f\" pack_scale_warn=\"no\"";
|
||||
else
|
||||
s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
case TYPE_SUBNET:
|
||||
case TYPE_ADDR:
|
||||
case TYPE_ENUM:
|
||||
case TYPE_STRING:
|
||||
case TYPE_FILE:
|
||||
case TYPE_TABLE:
|
||||
case TYPE_VECTOR:
|
||||
return "pack_unique=\"yes\"";
|
||||
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
DataSeries::DataSeries(WriterFrontend* frontend) : WriterBackend(frontend)
|
||||
{
|
||||
ds_compression = string((const char *)BifConst::LogDataSeries::compression->Bytes(),
|
||||
BifConst::LogDataSeries::compression->Len());
|
||||
ds_dump_schema = BifConst::LogDataSeries::dump_schema;
|
||||
ds_extent_size = BifConst::LogDataSeries::extent_size;
|
||||
ds_num_threads = BifConst::LogDataSeries::num_threads;
|
||||
ds_use_integer_for_time = BifConst::LogDataSeries::use_integer_for_time;
|
||||
ds_set_separator = ",";
|
||||
}
|
||||
|
||||
DataSeries::~DataSeries()
|
||||
{
|
||||
}
|
||||
|
||||
bool DataSeries::OpenLog(string path)
|
||||
{
|
||||
log_file = new DataSeriesSink(path + ".ds", compress_type);
|
||||
log_file->writeExtentLibrary(log_types);
|
||||
|
||||
for( size_t i = 0; i < schema_list.size(); ++i )
|
||||
extents.insert(std::make_pair(schema_list[i].field_name,
|
||||
GeneralField::create(log_series, schema_list[i].field_name)));
|
||||
|
||||
if ( ds_extent_size < ROW_MIN )
|
||||
{
|
||||
Warning(Fmt("%d is not a valid value for 'rows'. Using min of %d instead", (int)ds_extent_size, (int)ROW_MIN));
|
||||
ds_extent_size = ROW_MIN;
|
||||
}
|
||||
|
||||
else if( ds_extent_size > ROW_MAX )
|
||||
{
|
||||
Warning(Fmt("%d is not a valid value for 'rows'. Using max of %d instead", (int)ds_extent_size, (int)ROW_MAX));
|
||||
ds_extent_size = ROW_MAX;
|
||||
}
|
||||
|
||||
log_output = new OutputModule(*log_file, log_series, log_type, ds_extent_size);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DataSeries::DoInit(string path, int num_fields, const threading::Field* const * fields)
|
||||
{
|
||||
// We first construct an XML schema thing (and, if ds_dump_schema is
|
||||
// set, dump it to path + ".ds.xml"). Assuming that goes well, we
|
||||
// use that schema to build our output logfile and prepare it to be
|
||||
// written to.
|
||||
|
||||
// Note: compressor count must be set *BEFORE* DataSeriesSink is
|
||||
// instantiated.
|
||||
if( ds_num_threads < THREAD_MIN && ds_num_threads != 0 )
|
||||
{
|
||||
Warning(Fmt("%d is too few threads! Using %d instead", (int)ds_num_threads, (int)THREAD_MIN));
|
||||
ds_num_threads = THREAD_MIN;
|
||||
}
|
||||
|
||||
if( ds_num_threads > THREAD_MAX )
|
||||
{
|
||||
Warning(Fmt("%d is too many threads! Dropping back to %d", (int)ds_num_threads, (int)THREAD_MAX));
|
||||
ds_num_threads = THREAD_MAX;
|
||||
}
|
||||
|
||||
if( ds_num_threads > 0 )
|
||||
DataSeriesSink::setCompressorCount(ds_num_threads);
|
||||
|
||||
for ( int i = 0; i < num_fields; i++ )
|
||||
{
|
||||
const threading::Field* field = fields[i];
|
||||
SchemaValue val;
|
||||
val.ds_type = GetDSFieldType(field);
|
||||
val.field_name = string(field->name);
|
||||
val.field_options = GetDSOptionsForType(field);
|
||||
val.bro_type = field->TypeName();
|
||||
schema_list.push_back(val);
|
||||
}
|
||||
|
||||
string schema = BuildDSSchemaFromFieldTypes(schema_list, path);
|
||||
|
||||
if( ds_dump_schema )
|
||||
{
|
||||
FILE* pFile = fopen ( string(path + ".ds.xml").c_str() , "wb" );
|
||||
|
||||
if( pFile )
|
||||
{
|
||||
fwrite(schema.c_str(), 1, schema.length(), pFile);
|
||||
fclose(pFile);
|
||||
}
|
||||
|
||||
else
|
||||
Error(Fmt("cannot dump schema: %s", strerror(errno)));
|
||||
}
|
||||
|
||||
compress_type = Extent::compress_all;
|
||||
|
||||
if( ds_compression == "lzf" )
|
||||
compress_type = Extent::compress_lzf;
|
||||
|
||||
else if( ds_compression == "lzo" )
|
||||
compress_type = Extent::compress_lzo;
|
||||
|
||||
else if( ds_compression == "gz" )
|
||||
compress_type = Extent::compress_gz;
|
||||
|
||||
else if( ds_compression == "bz2" )
|
||||
compress_type = Extent::compress_bz2;
|
||||
|
||||
else if( ds_compression == "none" )
|
||||
compress_type = Extent::compress_none;
|
||||
|
||||
else if( ds_compression == "any" )
|
||||
compress_type = Extent::compress_all;
|
||||
|
||||
else
|
||||
Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str()));
|
||||
|
||||
log_type = log_types.registerTypePtr(schema);
|
||||
log_series.setType(log_type);
|
||||
|
||||
return OpenLog(path);
|
||||
}
|
||||
|
||||
bool DataSeries::DoFlush()
|
||||
{
|
||||
// Flushing is handled by DataSeries automatically, so this function
|
||||
// doesn't do anything.
|
||||
return true;
|
||||
}
|
||||
|
||||
void DataSeries::CloseLog()
|
||||
{
|
||||
for( ExtentIterator iter = extents.begin(); iter != extents.end(); ++iter )
|
||||
delete iter->second;
|
||||
|
||||
extents.clear();
|
||||
|
||||
// Don't delete the file before you delete the output, or bad things
|
||||
// will happen.
|
||||
delete log_output;
|
||||
delete log_file;
|
||||
|
||||
log_output = 0;
|
||||
log_file = 0;
|
||||
}
|
||||
|
||||
bool DataSeries::DoFinish()
|
||||
{
|
||||
CloseLog();
|
||||
|
||||
return WriterBackend::DoFinish();
|
||||
}
|
||||
|
||||
bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
|
||||
threading::Value** vals)
|
||||
{
|
||||
log_output->newRecord();
|
||||
|
||||
for( size_t i = 0; i < (size_t)num_fields; ++i )
|
||||
{
|
||||
ExtentIterator iter = extents.find(fields[i]->name);
|
||||
assert(iter != extents.end());
|
||||
|
||||
if( iter != extents.end() )
|
||||
{
|
||||
GeneralField *cField = iter->second;
|
||||
|
||||
if( vals[i]->present )
|
||||
cField->set(LogValueToString(vals[i]));
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DataSeries::DoRotate(string rotated_path, double open, double close, bool terminating)
|
||||
{
|
||||
// Note that if DS files are rotated too often, the aggregate log
|
||||
// size will be (much) larger.
|
||||
CloseLog();
|
||||
|
||||
string dsname = Path() + ".ds";
|
||||
string nname = rotated_path + ".ds";
|
||||
rename(dsname.c_str(), nname.c_str());
|
||||
|
||||
if ( ! FinishedRotation(nname, dsname, open, close, terminating) )
|
||||
{
|
||||
Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return OpenLog(Path());
|
||||
}
|
||||
|
||||
bool DataSeries::DoSetBuf(bool enabled)
|
||||
{
|
||||
// DataSeries is *always* buffered to some degree. This option is ignored.
|
||||
return true;
|
||||
}
|
||||
124
src/logging/writers/DataSeries.h
Normal file
124
src/logging/writers/DataSeries.h
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
// See the file "COPYING" in the main distribution directory for copyright.
|
||||
//
|
||||
// A binary log writer producing DataSeries output. See doc/data-series.rst
|
||||
// for more information.
|
||||
|
||||
#ifndef LOGGING_WRITER_DATA_SERIES_H
|
||||
#define LOGGING_WRITER_DATA_SERIES_H
|
||||
|
||||
#include <DataSeries/ExtentType.hpp>
|
||||
#include <DataSeries/DataSeriesFile.hpp>
|
||||
#include <DataSeries/DataSeriesModule.hpp>
|
||||
#include <DataSeries/GeneralField.hpp>
|
||||
|
||||
#include "../WriterBackend.h"
|
||||
|
||||
namespace logging { namespace writer {
|
||||
|
||||
class DataSeries : public WriterBackend {
|
||||
public:
|
||||
DataSeries(WriterFrontend* frontend);
|
||||
~DataSeries();
|
||||
|
||||
static WriterBackend* Instantiate(WriterFrontend* frontend)
|
||||
{ return new DataSeries(frontend); }
|
||||
|
||||
protected:
|
||||
// Overidden from WriterBackend.
|
||||
|
||||
virtual bool DoInit(string path, int num_fields,
|
||||
const threading::Field* const * fields);
|
||||
|
||||
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
|
||||
threading::Value** vals);
|
||||
virtual bool DoSetBuf(bool enabled);
|
||||
virtual bool DoRotate(string rotated_path, double open,
|
||||
double close, bool terminating);
|
||||
virtual bool DoFlush();
|
||||
virtual bool DoFinish();
|
||||
|
||||
private:
|
||||
static const size_t ROW_MIN = 2048; // Minimum extent size.
|
||||
static const size_t ROW_MAX = (1024 * 1024 * 100); // Maximum extent size.
|
||||
static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn.
|
||||
static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn.
|
||||
static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers.
|
||||
const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE.
|
||||
|
||||
struct SchemaValue
|
||||
{
|
||||
string ds_type;
|
||||
string bro_type;
|
||||
string field_name;
|
||||
string field_options;
|
||||
};
|
||||
|
||||
/**
|
||||
* Turns a log value into a std::string. Uses an ostringstream to do the
|
||||
* heavy lifting, but still need to switch on the type to know which value
|
||||
* in the union to give to the string string for processing.
|
||||
*
|
||||
* @param val The value we wish to convert to a string
|
||||
* @return the string value of val
|
||||
*/
|
||||
std::string LogValueToString(threading::Value *val);
|
||||
|
||||
/**
|
||||
* Takes a field type and converts it to a relevant DataSeries type.
|
||||
*
|
||||
* @param field We extract the type from this and convert it into a relevant DS type.
|
||||
* @return String representation of type that DataSeries can understand.
|
||||
*/
|
||||
string GetDSFieldType(const threading::Field *field);
|
||||
|
||||
/**
|
||||
* Are there any options we should put into the XML schema?
|
||||
*
|
||||
* @param field We extract the type from this and return any options that make sense for that type.
|
||||
* @return Options that can be added directly to the XML (e.g. "pack_relative=\"yes\"")
|
||||
*/
|
||||
std::string GetDSOptionsForType(const threading::Field *field);
|
||||
|
||||
/**
|
||||
* Takes a list of types, a list of names, and a title, and uses it to construct a valid DataSeries XML schema
|
||||
* thing, which is then returned as a std::string
|
||||
*
|
||||
* @param opts std::vector of strings containing a list of options to be appended to each field (e.g. "pack_relative=yes")
|
||||
* @param sTitle Name of this schema. Ideally, these schemas would be aggregated and re-used.
|
||||
*/
|
||||
string BuildDSSchemaFromFieldTypes(const vector<SchemaValue>& vals, string sTitle);
|
||||
|
||||
/** Closes the currently open file. */
|
||||
void CloseLog();
|
||||
|
||||
/** Opens a new file. */
|
||||
bool OpenLog(string path);
|
||||
|
||||
typedef std::map<string, GeneralField *> ExtentMap;
|
||||
typedef ExtentMap::iterator ExtentIterator;
|
||||
|
||||
// Internal DataSeries structures we need to keep track of.
|
||||
vector<SchemaValue> schema_list;
|
||||
ExtentTypeLibrary log_types;
|
||||
ExtentType::Ptr log_type;
|
||||
ExtentSeries log_series;
|
||||
ExtentMap extents;
|
||||
int compress_type;
|
||||
|
||||
DataSeriesSink* log_file;
|
||||
OutputModule* log_output;
|
||||
|
||||
// Options set from the script-level.
|
||||
uint64 ds_extent_size;
|
||||
uint64 ds_num_threads;
|
||||
string ds_compression;
|
||||
bool ds_dump_schema;
|
||||
bool ds_use_integer_for_time;
|
||||
string ds_set_separator;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
21
src/main.cc
21
src/main.cc
|
|
@ -203,6 +203,27 @@ void usage()
|
|||
fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str());
|
||||
fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n");
|
||||
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, " Supported log formats: ");
|
||||
|
||||
bool first = true;
|
||||
list<string> fmts = logging::Manager::SupportedFormats();
|
||||
|
||||
for ( list<string>::const_iterator i = fmts.begin(); i != fmts.end(); ++i )
|
||||
{
|
||||
if ( *i == "None" )
|
||||
// Skip, it's uninteresting.
|
||||
continue;
|
||||
|
||||
if ( ! first )
|
||||
fprintf(stderr, ",");
|
||||
|
||||
fprintf(stderr, "%s", (*i).c_str());
|
||||
first = false;
|
||||
}
|
||||
|
||||
fprintf(stderr, "\n");
|
||||
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,12 @@ public:
|
|||
*/
|
||||
int NumThreads() const { return all_threads.size(); }
|
||||
|
||||
/** Manually triggers processing of any thread input. This can be useful
|
||||
* if the main thread is waiting for a specific message from a child.
|
||||
* Usually, though, one should avoid using it.
|
||||
*/
|
||||
void ForceProcessing() { Process(); }
|
||||
|
||||
protected:
|
||||
friend class BasicThread;
|
||||
friend class MsgThread;
|
||||
|
|
|
|||
|
|
@ -10,13 +10,21 @@ namespace threading {
|
|||
|
||||
////// Messages.
|
||||
|
||||
// Signals child thread to terminate. This is actually a no-op; its only
|
||||
// purpose is unblock the current read operation so that the child's Run()
|
||||
// methods can check the termination status.
|
||||
class TerminateMessage : public InputMessage<MsgThread>
|
||||
// Signals child thread to shutdown operation.
|
||||
class FinishMessage : public InputMessage<MsgThread>
|
||||
{
|
||||
public:
|
||||
TerminateMessage(MsgThread* thread) : InputMessage<MsgThread>("Terminate", thread) { }
|
||||
FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread) { }
|
||||
|
||||
virtual bool Process() { return Object()->DoFinish(); }
|
||||
};
|
||||
|
||||
// A dummy message that's only purpose is unblock the current read operation
|
||||
// so that the child's Run() methods can check the termination status.
|
||||
class UnblockMessage : public InputMessage<MsgThread>
|
||||
{
|
||||
public:
|
||||
UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread) { }
|
||||
|
||||
virtual bool Process() { return true; }
|
||||
};
|
||||
|
|
@ -130,13 +138,29 @@ bool ReporterMessage::Process()
|
|||
MsgThread::MsgThread() : BasicThread()
|
||||
{
|
||||
cnt_sent_in = cnt_sent_out = 0;
|
||||
finished = false;
|
||||
thread_mgr->AddMsgThread(this);
|
||||
}
|
||||
|
||||
void MsgThread::OnStop()
|
||||
{
|
||||
// This is to unblock the current queue read operation.
|
||||
SendIn(new TerminateMessage(this), true);
|
||||
// Signal thread to terminate and wait until it has acknowledged.
|
||||
SendIn(new FinishMessage(this), true);
|
||||
|
||||
int cnt = 0;
|
||||
while ( ! finished )
|
||||
{
|
||||
if ( ++cnt > 1000 ) // Insurance against broken threads ...
|
||||
{
|
||||
reporter->Warning("thread %s didn't finish in time", Name().c_str());
|
||||
break;
|
||||
}
|
||||
|
||||
usleep(1000);
|
||||
}
|
||||
|
||||
// One more message to make sure the current queue read operation unblocks.
|
||||
SendIn(new UnblockMessage(this), true);
|
||||
}
|
||||
|
||||
void MsgThread::Heartbeat()
|
||||
|
|
@ -157,6 +181,14 @@ bool MsgThread::DoHeartbeat(double network_time, double current_time)
|
|||
return true;
|
||||
}
|
||||
|
||||
bool MsgThread::DoFinish()
|
||||
{
|
||||
// This is thread-safe "enough", we're the only one ever writing
|
||||
// there.
|
||||
finished = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void MsgThread::Info(const char* msg)
|
||||
{
|
||||
SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg));
|
||||
|
|
@ -189,7 +221,9 @@ void MsgThread::InternalWarning(const char* msg)
|
|||
|
||||
void MsgThread::InternalError(const char* msg)
|
||||
{
|
||||
SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg));
|
||||
// This one aborts immediately.
|
||||
fprintf(stderr, "internal error in thread: %s\n", msg);
|
||||
abort();
|
||||
}
|
||||
|
||||
#ifdef DEBUG
|
||||
|
|
|
|||
|
|
@ -171,6 +171,8 @@ public:
|
|||
protected:
|
||||
friend class Manager;
|
||||
friend class HeartbeatMessage;
|
||||
friend class FinishMessage;
|
||||
friend class FinishedMessage;
|
||||
|
||||
/**
|
||||
* Pops a message sent by the child from the child-to-main queue.
|
||||
|
|
@ -215,6 +217,12 @@ protected:
|
|||
*/
|
||||
virtual bool DoHeartbeat(double network_time, double current_time);
|
||||
|
||||
/** Triggered for execution in the child thread just before shutting threads down.
|
||||
* The child thread should finish its operations and then *must*
|
||||
* call this class' implementation.
|
||||
*/
|
||||
virtual bool DoFinish();
|
||||
|
||||
private:
|
||||
/**
|
||||
* Pops a message sent by the main thread from the main-to-chold
|
||||
|
|
@ -270,6 +278,8 @@ private:
|
|||
|
||||
uint64_t cnt_sent_in; // Counts message sent to child.
|
||||
uint64_t cnt_sent_out; // Counts message sent by child.
|
||||
|
||||
bool finished; // Set to true by Finished message.
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -24,6 +24,20 @@ bool Field::Write(SerializationFormat* fmt) const
|
|||
return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
|
||||
}
|
||||
|
||||
string Field::TypeName() const
|
||||
{
|
||||
string n = type_name(type);
|
||||
|
||||
if ( (type == TYPE_TABLE) || (type == TYPE_VECTOR) )
|
||||
{
|
||||
n += "[";
|
||||
n += type_name(subtype);
|
||||
n += "]";
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
Value::~Value()
|
||||
{
|
||||
if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@ struct Field {
|
|||
* @return False if an error occured.
|
||||
*/
|
||||
bool Write(SerializationFormat* fmt) const;
|
||||
|
||||
/**
|
||||
* Returns a textual description of the field's type. This method is
|
||||
* thread-safe.
|
||||
*/
|
||||
string TypeName() const;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -132,8 +138,8 @@ struct Value {
|
|||
|
||||
/**
|
||||
* Returns true if the type can be represented by a Value. If
|
||||
* `atomic_only` is true, will not permit composite types.
|
||||
*/
|
||||
* `atomic_only` is true, will not permit composite types. This
|
||||
* method is thread-safe. */
|
||||
static bool IsCompatibleType(BroType* t, bool atomic_only=false);
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -162,6 +162,7 @@ enum Writer %{
|
|||
WRITER_DEFAULT,
|
||||
WRITER_NONE,
|
||||
WRITER_ASCII,
|
||||
WRITER_DATASERIES,
|
||||
%}
|
||||
|
||||
enum ID %{
|
||||
|
|
|
|||
|
|
@ -19,4 +19,5 @@ scripts/base/init-bare.bro
|
|||
scripts/base/frameworks/logging/./postprocessors/./scp.bro
|
||||
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
|
||||
scripts/base/frameworks/logging/./writers/ascii.bro
|
||||
scripts/base/frameworks/logging/./writers/dataseries.bro
|
||||
scripts/policy/misc/loaded-scripts.bro
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ scripts/base/init-bare.bro
|
|||
scripts/base/frameworks/logging/./postprocessors/./scp.bro
|
||||
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
|
||||
scripts/base/frameworks/logging/./writers/ascii.bro
|
||||
scripts/base/frameworks/logging/./writers/dataseries.bro
|
||||
scripts/base/init-default.bro
|
||||
scripts/base/utils/site.bro
|
||||
scripts/base/utils/./patterns.bro
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
<field type="variable32" name="status" pack_unique="yes"/>
|
||||
<field type="variable32" name="country" pack_unique="yes"/>
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
<!-- status : string -->
|
||||
<!-- country : string -->
|
||||
|
|
@ -0,0 +1,290 @@
|
|||
test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries
|
||||
test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries
|
||||
test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries
|
||||
test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries
|
||||
test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries
|
||||
test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries
|
||||
test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries
|
||||
test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries
|
||||
test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries
|
||||
test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries
|
||||
> test.2011-03-07-03-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299466805.000000 10.0.0.1 20 10.0.0.2 1024
|
||||
1299470395.000000 10.0.0.2 20 10.0.0.3 0
|
||||
> test.2011-03-07-04-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299470405.000000 10.0.0.1 20 10.0.0.2 1025
|
||||
1299473995.000000 10.0.0.2 20 10.0.0.3 1
|
||||
> test.2011-03-07-05-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299474005.000000 10.0.0.1 20 10.0.0.2 1026
|
||||
1299477595.000000 10.0.0.2 20 10.0.0.3 2
|
||||
> test.2011-03-07-06-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299477605.000000 10.0.0.1 20 10.0.0.2 1027
|
||||
1299481195.000000 10.0.0.2 20 10.0.0.3 3
|
||||
> test.2011-03-07-07-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299481205.000000 10.0.0.1 20 10.0.0.2 1028
|
||||
1299484795.000000 10.0.0.2 20 10.0.0.3 4
|
||||
> test.2011-03-07-08-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299484805.000000 10.0.0.1 20 10.0.0.2 1029
|
||||
1299488395.000000 10.0.0.2 20 10.0.0.3 5
|
||||
> test.2011-03-07-09-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299488405.000000 10.0.0.1 20 10.0.0.2 1030
|
||||
1299491995.000000 10.0.0.2 20 10.0.0.3 6
|
||||
> test.2011-03-07-10-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299492005.000000 10.0.0.1 20 10.0.0.2 1031
|
||||
1299495595.000000 10.0.0.2 20 10.0.0.3 7
|
||||
> test.2011-03-07-11-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299495605.000000 10.0.0.1 20 10.0.0.2 1032
|
||||
1299499195.000000 10.0.0.2 20 10.0.0.3 8
|
||||
> test.2011-03-07-12-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
|
||||
# Extent, type='test'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||
1299499205.000000 10.0.0.1 20 10.0.0.2 1033
|
||||
1299502795.000000 10.0.0.2 20 10.0.0.3 9
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
<field type="variable32" name="status" pack_unique="yes"/>
|
||||
<field type="variable32" name="country" pack_unique="yes"/>
|
||||
</ExtentType>
|
||||
<!-- t : time -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
<!-- status : string -->
|
||||
<!-- country : string -->
|
||||
|
||||
# Extent, type='ssh'
|
||||
t id.orig_h id.orig_p id.resp_h id.resp_p status country
|
||||
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success unknown
|
||||
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure US
|
||||
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure UK
|
||||
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success BR
|
||||
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure MX
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="conn" version="1.0" namespace="bro-ids.org">
|
||||
<field type="int64" name="ts" pack_relative="ts" units="microseconds" epoch="unix"/>
|
||||
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
<field type="variable32" name="proto" pack_unique="yes"/>
|
||||
<field type="variable32" name="service" pack_unique="yes"/>
|
||||
<field type="int64" name="duration" pack_relative="duration" units="microseconds" epoch="unix"/>
|
||||
<field type="int64" name="orig_bytes" />
|
||||
<field type="int64" name="resp_bytes" />
|
||||
<field type="variable32" name="conn_state" pack_unique="yes"/>
|
||||
<field type="bool" name="local_orig" />
|
||||
<field type="int64" name="missed_bytes" />
|
||||
<field type="variable32" name="history" pack_unique="yes"/>
|
||||
<field type="int64" name="orig_pkts" />
|
||||
<field type="int64" name="orig_ip_bytes" />
|
||||
<field type="int64" name="resp_pkts" />
|
||||
<field type="int64" name="resp_ip_bytes" />
|
||||
</ExtentType>
|
||||
<!-- ts : time -->
|
||||
<!-- uid : string -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
<!-- proto : enum -->
|
||||
<!-- service : string -->
|
||||
<!-- duration : interval -->
|
||||
<!-- orig_bytes : count -->
|
||||
<!-- resp_bytes : count -->
|
||||
<!-- conn_state : string -->
|
||||
<!-- local_orig : bool -->
|
||||
<!-- missed_bytes : count -->
|
||||
<!-- history : string -->
|
||||
<!-- orig_pkts : count -->
|
||||
<!-- orig_ip_bytes : count -->
|
||||
<!-- resp_pkts : count -->
|
||||
<!-- resp_ip_bytes : count -->
|
||||
|
||||
# Extent, type='conn'
|
||||
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||
1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0
|
||||
1300475167097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0
|
||||
1300475167099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0
|
||||
1300475168853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 435 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 420 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 391 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 317 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 343 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 375 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 339 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 334 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 420 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 384 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 316 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 422 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 402 0 131 SHR F 0 Cd 0 0 1 159
|
||||
1300475168902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 374 0 198 SHR F 0 Cd 0 0 1 226
|
||||
1300475169899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0
|
||||
1300475170862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2613016 350 0 S0 F 0 D 7 546 0 0
|
||||
1300475171675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 100096 66 0 S0 F 0 D 2 162 0 0
|
||||
1300475171677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 100020 66 0 S0 F 0 D 2 122 0 0
|
||||
1300475173116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 99801 66 0 S0 F 0 D 2 162 0 0
|
||||
1300475173117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 99848 66 0 S0 F 0 D 2 122 0 0
|
||||
1300475173153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0
|
||||
1300475168859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 215893 1130 734 S1 F 1130 ShACad 4 216 4 950
|
||||
1300475168652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 61328 0 350 OTH F 0 CdA 1 52 1 402
|
||||
1300475168895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 227283 1178 734 S1 F 1178 ShACad 4 216 4 950
|
||||
1300475168902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 120040 534 412 S1 F 534 ShACad 3 164 3 576
|
||||
1300475168892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 229603 1148 734 S1 F 1148 ShACad 4 216 4 950
|
||||
1300475168855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 218501 1171 733 S1 F 1171 ShACad 4 216 4 949
|
||||
1300475168892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 220960 1137 733 S1 F 1137 ShACad 4 216 4 949
|
||||
1300475169780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48
|
||||
1300475168724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 119904 525 232 S1 F 525 ShACad 3 164 3 396
|
||||
1300475168855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 219720 1125 734 S1 F 1125 ShACad 4 216 4 950
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="conn" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
<field type="variable32" name="proto" pack_unique="yes"/>
|
||||
<field type="variable32" name="service" pack_unique="yes"/>
|
||||
<field type="double" name="duration" pack_relative="duration" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="int64" name="orig_bytes" />
|
||||
<field type="int64" name="resp_bytes" />
|
||||
<field type="variable32" name="conn_state" pack_unique="yes"/>
|
||||
<field type="bool" name="local_orig" />
|
||||
<field type="int64" name="missed_bytes" />
|
||||
<field type="variable32" name="history" pack_unique="yes"/>
|
||||
<field type="int64" name="orig_pkts" />
|
||||
<field type="int64" name="orig_ip_bytes" />
|
||||
<field type="int64" name="resp_pkts" />
|
||||
<field type="int64" name="resp_ip_bytes" />
|
||||
</ExtentType>
|
||||
<!-- ts : time -->
|
||||
<!-- uid : string -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
<!-- proto : enum -->
|
||||
<!-- service : string -->
|
||||
<!-- duration : interval -->
|
||||
<!-- orig_bytes : count -->
|
||||
<!-- resp_bytes : count -->
|
||||
<!-- conn_state : string -->
|
||||
<!-- local_orig : bool -->
|
||||
<!-- missed_bytes : count -->
|
||||
<!-- history : string -->
|
||||
<!-- orig_pkts : count -->
|
||||
<!-- orig_ip_bytes : count -->
|
||||
<!-- resp_pkts : count -->
|
||||
<!-- resp_ip_bytes : count -->
|
||||
|
||||
# Extent, type='conn'
|
||||
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||
1300475167.096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
|
||||
1300475167.097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
|
||||
1300475167.099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
|
||||
1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168.857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168.858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168.858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168.891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168.892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168.892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168.893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 0 89 SHR F 0 Cd 0 0 1 117
|
||||
1300475168.894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 0 99 SHR F 0 Cd 0 0 1 127
|
||||
1300475168.894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 0 183 SHR F 0 Cd 0 0 1 211
|
||||
1300475168.901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 0 131 SHR F 0 Cd 0 0 1 159
|
||||
1300475168.902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 0 198 SHR F 0 Cd 0 0 1 226
|
||||
1300475169.899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0
|
||||
1300475170.862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0
|
||||
1300475171.675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0
|
||||
1300475171.677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 F 0 D 2 122 0 0
|
||||
1300475173.116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0
|
||||
1300475173.117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0
|
||||
1300475173.153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0
|
||||
1300475168.859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0.215893 1130 734 S1 F 1130 ShACad 4 216 4 950
|
||||
1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 0 350 OTH F 0 CdA 1 52 1 402
|
||||
1300475168.895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0.227284 1178 734 S1 F 1178 ShACad 4 216 4 950
|
||||
1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0.120041 534 412 S1 F 534 ShACad 3 164 3 576
|
||||
1300475168.892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0.229603 1148 734 S1 F 1148 ShACad 4 216 4 950
|
||||
1300475168.855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0.218501 1171 733 S1 F 1171 ShACad 4 216 4 949
|
||||
1300475168.892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0.220961 1137 733 S1 F 1137 ShACad 4 216 4 949
|
||||
1300475169.780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48
|
||||
1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0.119905 525 232 S1 F 525 ShACad 3 164 3 396
|
||||
1300475168.855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0.219720 1125 734 S1 F 1125 ShACad 4 216 4 950
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
<field type="int64" name="offset" />
|
||||
<field type="variable32" name="extenttype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="DataSeries: XmlType">
|
||||
<field type="variable32" name="xmltype" />
|
||||
</ExtentType>
|
||||
|
||||
<ExtentType name="http" version="1.0" namespace="bro-ids.org">
|
||||
<field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.resp_p" />
|
||||
<field type="int64" name="trans_depth" />
|
||||
<field type="variable32" name="method" pack_unique="yes"/>
|
||||
<field type="variable32" name="host" pack_unique="yes"/>
|
||||
<field type="variable32" name="uri" pack_unique="yes"/>
|
||||
<field type="variable32" name="referrer" pack_unique="yes"/>
|
||||
<field type="variable32" name="user_agent" pack_unique="yes"/>
|
||||
<field type="int64" name="request_body_len" />
|
||||
<field type="int64" name="response_body_len" />
|
||||
<field type="int64" name="status_code" />
|
||||
<field type="variable32" name="status_msg" pack_unique="yes"/>
|
||||
<field type="int64" name="info_code" />
|
||||
<field type="variable32" name="info_msg" pack_unique="yes"/>
|
||||
<field type="variable32" name="filename" pack_unique="yes"/>
|
||||
<field type="variable32" name="tags" pack_unique="yes"/>
|
||||
<field type="variable32" name="username" pack_unique="yes"/>
|
||||
<field type="variable32" name="password" pack_unique="yes"/>
|
||||
<field type="variable32" name="proxied" pack_unique="yes"/>
|
||||
<field type="variable32" name="mime_type" pack_unique="yes"/>
|
||||
<field type="variable32" name="md5" pack_unique="yes"/>
|
||||
<field type="variable32" name="extraction_file" pack_unique="yes"/>
|
||||
</ExtentType>
|
||||
<!-- ts : time -->
|
||||
<!-- uid : string -->
|
||||
<!-- id.orig_h : addr -->
|
||||
<!-- id.orig_p : port -->
|
||||
<!-- id.resp_h : addr -->
|
||||
<!-- id.resp_p : port -->
|
||||
<!-- trans_depth : count -->
|
||||
<!-- method : string -->
|
||||
<!-- host : string -->
|
||||
<!-- uri : string -->
|
||||
<!-- referrer : string -->
|
||||
<!-- user_agent : string -->
|
||||
<!-- request_body_len : count -->
|
||||
<!-- response_body_len : count -->
|
||||
<!-- status_code : count -->
|
||||
<!-- status_msg : string -->
|
||||
<!-- info_code : count -->
|
||||
<!-- info_msg : string -->
|
||||
<!-- filename : string -->
|
||||
<!-- tags : table[enum] -->
|
||||
<!-- username : string -->
|
||||
<!-- password : string -->
|
||||
<!-- proxied : table[string] -->
|
||||
<!-- mime_type : string -->
|
||||
<!-- md5 : string -->
|
||||
<!-- extraction_file : file -->
|
||||
|
||||
# Extent, type='http'
|
||||
ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
|
||||
1300475168.843894 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0
|
||||
1300475168.975800 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475168.976327 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475168.979160 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.012666 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.012730 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.014860 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.022665 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0
|
||||
1300475169.036294 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.036798 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.039923 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.074793 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.074938 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
1300475169.075065 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0
|
||||
1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0
|
||||
1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0
|
||||
1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0
|
||||
1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0
|
||||
1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0
|
||||
1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0
|
||||
1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0
|
||||
1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
|
||||
1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
|
||||
1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii
|
||||
1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii
|
||||
1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii
|
||||
1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii
|
||||
1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii
|
||||
1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii
|
||||
1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii
|
||||
1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii
|
||||
1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii
|
||||
1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0
|
||||
test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0
|
||||
test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0
|
||||
test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0
|
||||
test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0
|
||||
test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0
|
||||
test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0
|
||||
test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0
|
||||
test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
|
||||
test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
|
||||
test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii
|
||||
test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii
|
||||
test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii
|
||||
test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii
|
||||
test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii
|
||||
test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii
|
||||
test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii
|
||||
test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii
|
||||
test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii
|
||||
test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
|
||||
> test.2011-03-07-03-00-05.log
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
|
|
|
|||
35
testing/btest/core/leaks/dataseries-rotate.bro
Normal file
35
testing/btest/core/leaks/dataseries-rotate.bro
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
|
||||
#
|
||||
# @TEST-GROUP: leaks
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/rotation.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
|
||||
module Test;
|
||||
|
||||
export {
|
||||
# Create a new ID for our log stream
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
# Define a record with all the columns the log file can have.
|
||||
# (I'm using a subset of fields from ssh-ext for demonstration.)
|
||||
type Log: record {
|
||||
t: time;
|
||||
id: conn_id; # Will be rolled out into individual columns.
|
||||
} &log;
|
||||
}
|
||||
|
||||
redef Log::default_rotation_interval = 1hr;
|
||||
redef Log::default_rotation_postprocessor_cmd = "echo";
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
Log::create_stream(Test::LOG, [$columns=Log]);
|
||||
}
|
||||
|
||||
event new_connection(c: connection)
|
||||
{
|
||||
Log::write(Test::LOG, [$t=network_time(), $id=c$id]);
|
||||
}
|
||||
10
testing/btest/core/leaks/dataseries.bro
Normal file
10
testing/btest/core/leaks/dataseries.bro
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Needs perftools support.
|
||||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
|
||||
#
|
||||
# @TEST-GROUP: leaks
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
|
||||
# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: test -e ssh.ds.xml
|
||||
# @TEST-EXEC: btest-diff ssh.ds.xml
|
||||
|
||||
module SSH;
|
||||
|
||||
redef LogDataSeries::dump_schema = T;
|
||||
|
||||
# Haven't yet found a way to check for the effect of these.
|
||||
redef LogDataSeries::compression = "bz2";
|
||||
redef LogDataSeries::extent_size = 1000;
|
||||
redef LogDataSeries::num_threads = 5;
|
||||
|
||||
# LogDataSeries::use_integer_for_time is tested separately.
|
||||
|
||||
export {
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
type Log: record {
|
||||
t: time;
|
||||
id: conn_id; # Will be rolled out into individual columns.
|
||||
status: string &optional;
|
||||
country: string &default="unknown";
|
||||
} &log;
|
||||
}
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
Log::create_stream(SSH::LOG, [$columns=Log]);
|
||||
|
||||
local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
|
||||
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out
|
||||
# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt --skip-index $i; done >>out
|
||||
# @TEST-EXEC: btest-diff out
|
||||
|
||||
module Test;
|
||||
|
||||
export {
|
||||
# Create a new ID for our log stream
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
# Define a record with all the columns the log file can have.
|
||||
# (I'm using a subset of fields from ssh-ext for demonstration.)
|
||||
type Log: record {
|
||||
t: time;
|
||||
id: conn_id; # Will be rolled out into individual columns.
|
||||
} &log;
|
||||
}
|
||||
|
||||
redef Log::default_rotation_interval = 1hr;
|
||||
redef Log::default_rotation_postprocessor_cmd = "echo";
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
Log::create_stream(Test::LOG, [$columns=Log]);
|
||||
}
|
||||
|
||||
event new_connection(c: connection)
|
||||
{
|
||||
Log::write(Test::LOG, [$t=network_time(), $id=c$id]);
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: ds2txt --skip-index ssh.ds >ssh.ds.txt
|
||||
# @TEST-EXEC: btest-diff ssh.ds.txt
|
||||
|
||||
module SSH;
|
||||
|
||||
export {
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
type Log: record {
|
||||
t: time;
|
||||
id: conn_id; # Will be rolled out into individual columns.
|
||||
status: string &optional;
|
||||
country: string &default="unknown";
|
||||
} &log;
|
||||
}
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
Log::create_stream(SSH::LOG, [$columns=Log]);
|
||||
|
||||
local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
|
||||
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
|
||||
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt
|
||||
# @TEST-EXEC: btest-diff conn.ds.txt
|
||||
|
||||
redef LogDataSeries::use_integer_for_time = T;
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt
|
||||
# @TEST-EXEC: ds2txt --skip-index http.ds >http.ds.txt
|
||||
# @TEST-EXEC: btest-diff conn.ds.txt
|
||||
# @TEST-EXEC: btest-diff http.ds.txt
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#
|
||||
# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT | egrep "test|test2" | sort >out
|
||||
#@TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out
|
||||
# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
|
||||
# @TEST-EXEC: btest-diff out
|
||||
# @TEST-EXEC: btest-diff .stderr
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#
|
||||
# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT 2>&1 | grep "test" >out
|
||||
# @TEST-EXEC: for i in test.*.log; do printf '> %s\n' $i; cat $i; done >>out
|
||||
# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 | grep "test" >out
|
||||
# @TEST-EXEC: for i in `ls test.*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out
|
||||
# @TEST-EXEC: btest-diff out
|
||||
|
||||
module Test;
|
||||
|
|
|
|||
2
testing/external/subdir-btest.cfg
vendored
2
testing/external/subdir-btest.cfg
vendored
|
|
@ -10,7 +10,7 @@ BROPATH=`bash -c %(testbase)s/../../../build/bro-path-dev`:%(testbase)s/../scrip
|
|||
BRO_SEED_FILE=%(testbase)s/../random.seed
|
||||
TZ=UTC
|
||||
LC_ALL=C
|
||||
PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(default_path)s
|
||||
PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(testbase)s/../../scripts:%(default_path)s
|
||||
TEST_DIFF_CANONIFIER=%(testbase)s/../../scripts/diff-canonifier-external
|
||||
TEST_DIFF_BRIEF=1
|
||||
TRACES=%(testbase)s/Traces
|
||||
|
|
|
|||
6
testing/scripts/has-writer
Executable file
6
testing/scripts/has-writer
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#! /usr/bin/env bash
|
||||
#
|
||||
# Returns true if Bro has been compiled with support for writer type
|
||||
# $1. The type name must match what "bro --help" prints.
|
||||
|
||||
bro --helper 2>&1 | grep -qi "Supported log formats:.*$1"
|
||||
Loading…
Add table
Add a link
Reference in a new issue