mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Fix for when not producing local output; that hung.
* origin/topic/robin/dataseries: Moving trace for rotation test into traces directory. Fixing a rotation race condition at termination. Portability fixes. Extending DS docs with some examples. Updating doc. Fixing pack_scale and time-as-int. Adding format specifier to DS spec to print out double as %.6f. DataSeries updates and fixes. DataSeries tuning. Tweaking DataSeries support. Extending log post-processor call to include the name of the writer. Removing an unnecessary const cast. DataSeries TODO list with open issues/questions. Starting DataSeries HowTo. Additional test output canonification for ds2txt's timestamps. In threads, an internal error now immediately aborts. DataSeries cleanup. Working on DataSeries support. Merging in DataSeries support from topic/gilbert/logging. Fixing threads' DoFinish() method.
This commit is contained in:
Robin Sommer
commit
7cc863c5fc
52 changed files with 1844 additions and 133 deletions
|
|
@ -107,6 +107,21 @@ if (GOOGLEPERFTOOLS_FOUND)
|
||||||
endif ()
|
endif ()
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
|
set(USE_DATASERIES false)
|
||||||
|
find_package(Lintel)
|
||||||
|
find_package(DataSeries)
|
||||||
|
find_package(LibXML2)
|
||||||
|
|
||||||
|
if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
|
||||||
|
set(USE_DATASERIES true)
|
||||||
|
include_directories(BEFORE ${Lintel_INCLUDE_DIR})
|
||||||
|
include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
|
||||||
|
include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
|
||||||
|
list(APPEND OPTLIBS ${Lintel_LIBRARIES})
|
||||||
|
list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
|
||||||
|
list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
|
||||||
|
endif()
|
||||||
|
|
||||||
if (ENABLE_PERFTOOLS_DEBUG)
|
if (ENABLE_PERFTOOLS_DEBUG)
|
||||||
# Just a no op to prevent CMake from complaining about manually-specified
|
# Just a no op to prevent CMake from complaining about manually-specified
|
||||||
# ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
|
# ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
|
||||||
|
|
@ -198,6 +213,7 @@ message(
|
||||||
"\nGeoIP: ${USE_GEOIP}"
|
"\nGeoIP: ${USE_GEOIP}"
|
||||||
"\nGoogle perftools: ${USE_PERFTOOLS}"
|
"\nGoogle perftools: ${USE_PERFTOOLS}"
|
||||||
"\n debugging: ${USE_PERFTOOLS_DEBUG}"
|
"\n debugging: ${USE_PERFTOOLS_DEBUG}"
|
||||||
|
"\nDataSeries: ${USE_DATASERIES}"
|
||||||
"\n"
|
"\n"
|
||||||
"\n================================================================\n"
|
"\n================================================================\n"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
7
NEWS
7
NEWS
|
|
@ -47,6 +47,13 @@ Bro 2.1
|
||||||
joint set of events. The `icmp_conn` record got a new boolean field
|
joint set of events. The `icmp_conn` record got a new boolean field
|
||||||
'v6' that indicates whether the ICMP message is v4 or v6.
|
'v6' that indicates whether the ICMP message is v4 or v6.
|
||||||
|
|
||||||
|
- Log postprocessor scripts get an additional argument indicating the
|
||||||
|
type of the log writer in use (e.g., "ascii").
|
||||||
|
|
||||||
|
- BroControl's make-archive-name scripts also receives the writer
|
||||||
|
type, but as it's 2nd(!) argument. If you're using a custom version
|
||||||
|
of that script, you need to adapt it. See the shipped version for
|
||||||
|
details.
|
||||||
|
|
||||||
TODO: Extend.
|
TODO: Extend.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit 519d2e21ee375833c89eb6f7dc95c1eac3de17ab
|
Subproject commit ed933502b4d2518f94b6cfa7a5b371e53fda5c3d
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit 76876ce0e7da4888c91b3aea024c5cfd36405310
|
Subproject commit e0da8d0e284bbebbaef711c91c1b961580f225d2
|
||||||
2
cmake
2
cmake
|
|
@ -1 +1 @@
|
||||||
Subproject commit 49278736c1404cb8c077272b80312c947e68bf52
|
Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b
|
||||||
|
|
@ -114,6 +114,9 @@
|
||||||
/* Analyze Mobile IPv6 traffic */
|
/* Analyze Mobile IPv6 traffic */
|
||||||
#cmakedefine ENABLE_MOBILE_IPV6
|
#cmakedefine ENABLE_MOBILE_IPV6
|
||||||
|
|
||||||
|
/* Use the DataSeries writer. */
|
||||||
|
#cmakedefine USE_DATASERIES
|
||||||
|
|
||||||
/* Version number of package */
|
/* Version number of package */
|
||||||
#define VERSION "@VERSION@"
|
#define VERSION "@VERSION@"
|
||||||
|
|
||||||
|
|
|
||||||
9
configure
vendored
9
configure
vendored
|
|
@ -56,6 +56,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||||
--with-ruby-lib=PATH path to ruby library
|
--with-ruby-lib=PATH path to ruby library
|
||||||
--with-ruby-inc=PATH path to ruby headers
|
--with-ruby-inc=PATH path to ruby headers
|
||||||
--with-swig=PATH path to SWIG executable
|
--with-swig=PATH path to SWIG executable
|
||||||
|
--with-dataseries=PATH path to DataSeries and Lintel libraries
|
||||||
|
--with-xml2=PATH path to libxml2 installation (for DataSeries)
|
||||||
|
|
||||||
Packaging Options (for developers):
|
Packaging Options (for developers):
|
||||||
--binary-package toggle special logic for binary packaging
|
--binary-package toggle special logic for binary packaging
|
||||||
|
|
@ -213,6 +215,13 @@ while [ $# -ne 0 ]; do
|
||||||
--with-swig=*)
|
--with-swig=*)
|
||||||
append_cache_entry SWIG_EXECUTABLE PATH $optarg
|
append_cache_entry SWIG_EXECUTABLE PATH $optarg
|
||||||
;;
|
;;
|
||||||
|
--with-dataseries=*)
|
||||||
|
append_cache_entry DataSeries_ROOT_DIR PATH $optarg
|
||||||
|
append_cache_entry Lintel_ROOT_DIR PATH $optarg
|
||||||
|
;;
|
||||||
|
--with-xml2=*)
|
||||||
|
append_cache_entry LibXML2_ROOT_DIR PATH $optarg
|
||||||
|
;;
|
||||||
--binary-package)
|
--binary-package)
|
||||||
append_cache_entry BINARY_PACKAGING_MODE BOOL true
|
append_cache_entry BINARY_PACKAGING_MODE BOOL true
|
||||||
;;
|
;;
|
||||||
|
|
|
||||||
168
doc/logging-dataseries.rst
Normal file
168
doc/logging-dataseries.rst
Normal file
|
|
@ -0,0 +1,168 @@
|
||||||
|
|
||||||
|
=============================
|
||||||
|
Binary Output with DataSeries
|
||||||
|
=============================
|
||||||
|
|
||||||
|
.. rst-class:: opening
|
||||||
|
|
||||||
|
Bro's default ASCII log format is not exactly the most efficient
|
||||||
|
way for storing large volumes of data. An an alternative, Bro comes
|
||||||
|
with experimental support for `DataSeries
|
||||||
|
<http://www.hpl.hp.com/techreports/2009/HPL-2009-323.html>`_
|
||||||
|
output, an efficient binary format for recording structured bulk
|
||||||
|
data. DataSeries is developed and maintained at HP Labs.
|
||||||
|
|
||||||
|
.. contents::
|
||||||
|
|
||||||
|
Installing DataSeries
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
To use DataSeries, its libraries must be available at compile-time,
|
||||||
|
along with the supporting *Lintel* package. Generally, both are
|
||||||
|
distributed on `HP Labs' web site
|
||||||
|
<http://tesla.hpl.hp.com/opensource/>`_. Currently, however, you need
|
||||||
|
to use recent developments versions for both packages, which you can
|
||||||
|
download from github like this::
|
||||||
|
|
||||||
|
git clone http://github.com/dataseries/Lintel
|
||||||
|
git clone http://github.com/dataseries/DataSeries
|
||||||
|
|
||||||
|
To build and install the two into ``<prefix>``, do::
|
||||||
|
|
||||||
|
( cd Lintel && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
|
||||||
|
( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
|
||||||
|
|
||||||
|
Please refer to the packages' documentation for more information about
|
||||||
|
the installation process. In particular, there's more information on
|
||||||
|
required and optional `dependencies for Lintel
|
||||||
|
<https://raw.github.com/eric-anderson/Lintel/master/doc/dependencies.txt>`_
|
||||||
|
and `dependencies for DataSeries
|
||||||
|
<https://raw.github.com/eric-anderson/DataSeries/master/doc/dependencies.txt>`_
|
||||||
|
|
||||||
|
Compiling Bro with DataSeries Support
|
||||||
|
-------------------------------------
|
||||||
|
|
||||||
|
Once you have installed DataSeries, Bro's ``configure`` should pick it
|
||||||
|
up automatically as long as it finds it in a standard system location.
|
||||||
|
Alternatively, you can specify the DataSeries installation prefix
|
||||||
|
manually with ``--with-dataseries=<prefix>``. Keep an eye on
|
||||||
|
``configure``'s summary output, if it looks like the following, Bro
|
||||||
|
found DataSeries and will compile in the support::
|
||||||
|
|
||||||
|
# ./configure --with-dataseries=/usr/local
|
||||||
|
[...]
|
||||||
|
====================| Bro Build Summary |=====================
|
||||||
|
[...]
|
||||||
|
DataSeries: true
|
||||||
|
[...]
|
||||||
|
================================================================
|
||||||
|
|
||||||
|
Activating DataSeries
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
The direct way to use DataSeries is to switch *all* log files over to
|
||||||
|
the binary format. To do that, just add ``redef
|
||||||
|
Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``.
|
||||||
|
For testing, you can also just pass that on the command line::
|
||||||
|
|
||||||
|
bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
|
||||||
|
With that, Bro will now write all its output into DataSeries files
|
||||||
|
``*.ds``. You can inspect these using DataSeries's set of command line
|
||||||
|
tools, which its installation process installs into ``<prefix>/bin``.
|
||||||
|
For example, to convert a file back into an ASCII representation::
|
||||||
|
|
||||||
|
$ ds2txt conn.log
|
||||||
|
[... We skip a bunch of meta data here ...]
|
||||||
|
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||||
|
1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
|
||||||
|
1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
|
||||||
|
1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
|
||||||
|
1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117
|
||||||
|
1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127
|
||||||
|
1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211
|
||||||
|
[...]
|
||||||
|
|
||||||
|
(``--skip-all`` suppresses the meta data.)
|
||||||
|
|
||||||
|
Note that the ASCII conversion is *not* equivalent to Bro's default
|
||||||
|
output format.
|
||||||
|
|
||||||
|
You can also switch only individual files over to DataSeries by adding
|
||||||
|
code like this to your ``local.bro``::
|
||||||
|
|
||||||
|
.. code:: bro
|
||||||
|
|
||||||
|
event bro_init()
|
||||||
|
{
|
||||||
|
local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
|
||||||
|
f$writer = Log::WRITER_DATASERIES; # Change writer type.
|
||||||
|
Log::add_filter(Conn::LOG, f); # Replace filter with adapted version.
|
||||||
|
}
|
||||||
|
|
||||||
|
Bro's DataSeries writer comes with a few tuning options, see
|
||||||
|
:doc:`scripts/base/frameworks/logging/writers/dataseries`.
|
||||||
|
|
||||||
|
Working with DataSeries
|
||||||
|
=======================
|
||||||
|
|
||||||
|
Here are few examples of using DataSeries command line tools to work
|
||||||
|
with the output files.
|
||||||
|
|
||||||
|
* Printing CSV::
|
||||||
|
|
||||||
|
$ ds2txt --csv conn.log
|
||||||
|
ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes
|
||||||
|
1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0
|
||||||
|
1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0
|
||||||
|
1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0
|
||||||
|
1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0
|
||||||
|
[...]
|
||||||
|
|
||||||
|
Add ``--separator=X`` to set a different separator.
|
||||||
|
|
||||||
|
* Extracting a subset of columns::
|
||||||
|
|
||||||
|
$ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log
|
||||||
|
1258790493.773208 192.168.1.255 137
|
||||||
|
1258790451.402091 192.168.1.255 138
|
||||||
|
1258790493.787448 192.168.1.255 138
|
||||||
|
1258790615.268111 192.168.1.255 137
|
||||||
|
1258790615.289842 192.168.1.255 138
|
||||||
|
[...]
|
||||||
|
|
||||||
|
* Filtering rows::
|
||||||
|
|
||||||
|
$ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all conn.ds
|
||||||
|
1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp 15.004568 798 0 S0 F 0 D 6 966 0 0
|
||||||
|
1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp 15.004581 798 0 S0 F 0 D 6 966 0 0
|
||||||
|
1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp 15.005071 798 0 S0 F 0 D 6 966 0 0
|
||||||
|
1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp 15.004824 798 0 S0 F 0 D 6 966 0 0
|
||||||
|
1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp 15.005078 798 0 S0 F 0 D 6 966 0 0
|
||||||
|
1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp 15.004082 798 0 S0 F 0 D 6 966 0 0
|
||||||
|
[...]
|
||||||
|
|
||||||
|
* Calculate some statistics:
|
||||||
|
|
||||||
|
Mean/stdev/min/max over a column::
|
||||||
|
|
||||||
|
$ dsstatgroupby '*' basic duration from conn.ds
|
||||||
|
# Begin DSStatGroupByModule
|
||||||
|
# processed 2159 rows, where clause eliminated 0 rows
|
||||||
|
# count(*), mean(duration), stddev, min, max
|
||||||
|
2159, 42.7938, 1858.34, 0, 86370
|
||||||
|
[...]
|
||||||
|
|
||||||
|
Quantiles of total connection volume::
|
||||||
|
|
||||||
|
> dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds
|
||||||
|
[...]
|
||||||
|
2159 data points, mean 24616 +- 343295 [0,1.26615e+07]
|
||||||
|
quantiles about every 216 data points:
|
||||||
|
10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469
|
||||||
|
tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262
|
||||||
|
[...]
|
||||||
|
|
||||||
|
The ``man`` pages for these tool show further options, and their
|
||||||
|
``-h`` option gives some more information (either can be a bit cryptic
|
||||||
|
unfortunately though).
|
||||||
|
|
@ -36,6 +36,7 @@ rest_target(${psd} base/frameworks/logging/main.bro)
|
||||||
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
|
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
|
||||||
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
|
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
|
||||||
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
|
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
|
||||||
|
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
|
||||||
rest_target(${psd} base/frameworks/metrics/cluster.bro)
|
rest_target(${psd} base/frameworks/metrics/cluster.bro)
|
||||||
rest_target(${psd} base/frameworks/metrics/main.bro)
|
rest_target(${psd} base/frameworks/metrics/main.bro)
|
||||||
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
|
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
@load ./main
|
@load ./main
|
||||||
@load ./postprocessors
|
@load ./postprocessors
|
||||||
@load ./writers/ascii
|
@load ./writers/ascii
|
||||||
|
@load ./writers/dataseries
|
||||||
|
|
|
||||||
|
|
@ -376,13 +376,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo
|
||||||
if ( pp_cmd == "" )
|
if ( pp_cmd == "" )
|
||||||
return T;
|
return T;
|
||||||
|
|
||||||
|
# Turn, e.g., Log::WRITER_ASCII into "ascii".
|
||||||
|
local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", "");
|
||||||
|
|
||||||
# The date format is hard-coded here to provide a standardized
|
# The date format is hard-coded here to provide a standardized
|
||||||
# script interface.
|
# script interface.
|
||||||
system(fmt("%s %s %s %s %s %d",
|
system(fmt("%s %s %s %s %s %d %s",
|
||||||
pp_cmd, npath, info$path,
|
pp_cmd, npath, info$path,
|
||||||
strftime("%y-%m-%d_%H.%M.%S", info$open),
|
strftime("%y-%m-%d_%H.%M.%S", info$open),
|
||||||
strftime("%y-%m-%d_%H.%M.%S", info$close),
|
strftime("%y-%m-%d_%H.%M.%S", info$close),
|
||||||
info$terminating));
|
info$terminating, writer));
|
||||||
|
|
||||||
return T;
|
return T;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
60
scripts/base/frameworks/logging/writers/dataseries.bro
Normal file
60
scripts/base/frameworks/logging/writers/dataseries.bro
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
##! Interface for the DataSeries log writer.
|
||||||
|
|
||||||
|
module LogDataSeries;
|
||||||
|
|
||||||
|
export {
|
||||||
|
## Compression to use with the DS output file. Options are:
|
||||||
|
##
|
||||||
|
## 'none' -- No compression.
|
||||||
|
## 'lzf' -- LZF compression. Very quick, but leads to larger output files.
|
||||||
|
## 'lzo' -- LZO compression. Very fast decompression times.
|
||||||
|
## 'gz' -- GZIP compression. Slower than LZF, but also produces smaller output.
|
||||||
|
## 'bz2' -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
|
||||||
|
const compression = "lzo" &redef;
|
||||||
|
|
||||||
|
## The extent buffer size.
|
||||||
|
## Larger values here lead to better compression and more efficient writes, but
|
||||||
|
## also increase the lag between the time events are received and the time they
|
||||||
|
## are actually written to disk.
|
||||||
|
const extent_size = 65536 &redef;
|
||||||
|
|
||||||
|
## Should we dump the XML schema we use for this DS file to disk?
|
||||||
|
## If yes, the XML schema shares the name of the logfile, but has
|
||||||
|
## an XML ending.
|
||||||
|
const dump_schema = F &redef;
|
||||||
|
|
||||||
|
## How many threads should DataSeries spawn to perform compression?
|
||||||
|
## Note that this dictates the number of threads per log stream. If
|
||||||
|
## you're using a lot of streams, you may want to keep this number
|
||||||
|
## relatively small.
|
||||||
|
##
|
||||||
|
## Default value is 1, which will spawn one thread / stream.
|
||||||
|
##
|
||||||
|
## Maximum is 128, minimum is 1.
|
||||||
|
const num_threads = 1 &redef;
|
||||||
|
|
||||||
|
## Should time be stored as an integer or a double?
|
||||||
|
## Storing time as a double leads to possible precision issues and
|
||||||
|
## can (significantly) increase the size of the resulting DS log.
|
||||||
|
## That said, timestamps stored in double form are consistent
|
||||||
|
## with the rest of Bro, including the standard ASCII log. Hence, we
|
||||||
|
## use them by default.
|
||||||
|
const use_integer_for_time = F &redef;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Default function to postprocess a rotated DataSeries log file. It moves the
|
||||||
|
# rotated file to a new name that includes a timestamp with the opening time, and
|
||||||
|
# then runs the writer's default postprocessor command on it.
|
||||||
|
function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
|
||||||
|
{
|
||||||
|
# Move file to name including both opening and closing time.
|
||||||
|
local dst = fmt("%s.%s.ds", info$path,
|
||||||
|
strftime(Log::default_rotation_date_format, info$open));
|
||||||
|
|
||||||
|
system(fmt("/bin/mv %s %s", info$fname, dst));
|
||||||
|
|
||||||
|
# Run default postprocessor.
|
||||||
|
return Log::run_rotation_postprocessor_cmd(info, dst);
|
||||||
|
}
|
||||||
|
|
||||||
|
redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };
|
||||||
|
|
@ -417,6 +417,7 @@ set(bro_SRCS
|
||||||
logging/WriterBackend.cc
|
logging/WriterBackend.cc
|
||||||
logging/WriterFrontend.cc
|
logging/WriterFrontend.cc
|
||||||
logging/writers/Ascii.cc
|
logging/writers/Ascii.cc
|
||||||
|
logging/writers/DataSeries.cc
|
||||||
logging/writers/None.cc
|
logging/writers/None.cc
|
||||||
|
|
||||||
nb_dns.c
|
nb_dns.c
|
||||||
|
|
|
||||||
|
|
@ -15,10 +15,9 @@
|
||||||
|
|
||||||
extern int generate_documentation;
|
extern int generate_documentation;
|
||||||
|
|
||||||
|
// Note: This function must be thread-safe.
|
||||||
const char* type_name(TypeTag t)
|
const char* type_name(TypeTag t)
|
||||||
{
|
{
|
||||||
static char errbuf[512];
|
|
||||||
|
|
||||||
static const char* type_names[int(NUM_TYPES)] = {
|
static const char* type_names[int(NUM_TYPES)] = {
|
||||||
"void",
|
"void",
|
||||||
"bool", "int", "count", "counter",
|
"bool", "int", "count", "counter",
|
||||||
|
|
@ -37,10 +36,7 @@ const char* type_name(TypeTag t)
|
||||||
};
|
};
|
||||||
|
|
||||||
if ( int(t) >= NUM_TYPES )
|
if ( int(t) >= NUM_TYPES )
|
||||||
{
|
return "type_name(): not a type tag";
|
||||||
snprintf(errbuf, sizeof(errbuf), "%d: not a type tag", int(t));
|
|
||||||
return errbuf;
|
|
||||||
}
|
|
||||||
|
|
||||||
return type_names[int(t)];
|
return type_names[int(t)];
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -72,3 +72,12 @@ const set_separator: string;
|
||||||
const empty_field: string;
|
const empty_field: string;
|
||||||
const unset_field: string;
|
const unset_field: string;
|
||||||
|
|
||||||
|
# Options for the DataSeries writer.
|
||||||
|
|
||||||
|
module LogDataSeries;
|
||||||
|
|
||||||
|
const compression: string;
|
||||||
|
const extent_size: count;
|
||||||
|
const dump_schema: bool;
|
||||||
|
const use_integer_for_time: bool;
|
||||||
|
const num_threads: count;
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@
|
||||||
#include "../NetVar.h"
|
#include "../NetVar.h"
|
||||||
#include "../Net.h"
|
#include "../Net.h"
|
||||||
|
|
||||||
|
#include "threading/Manager.h"
|
||||||
#include "threading/SerialTypes.h"
|
#include "threading/SerialTypes.h"
|
||||||
|
|
||||||
#include "Manager.h"
|
#include "Manager.h"
|
||||||
|
|
@ -16,9 +17,11 @@
|
||||||
#include "writers/Ascii.h"
|
#include "writers/Ascii.h"
|
||||||
#include "writers/None.h"
|
#include "writers/None.h"
|
||||||
|
|
||||||
|
#ifdef USE_DATASERIES
|
||||||
|
#include "writers/DataSeries.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
using namespace logging;
|
using namespace logging;
|
||||||
using threading::Value;
|
|
||||||
using threading::Field;
|
|
||||||
|
|
||||||
// Structure describing a log writer type.
|
// Structure describing a log writer type.
|
||||||
struct WriterDefinition {
|
struct WriterDefinition {
|
||||||
|
|
@ -32,6 +35,9 @@ struct WriterDefinition {
|
||||||
WriterDefinition log_writers[] = {
|
WriterDefinition log_writers[] = {
|
||||||
{ BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate },
|
{ BifEnum::Log::WRITER_NONE, "None", 0, writer::None::Instantiate },
|
||||||
{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
|
{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
|
||||||
|
#ifdef USE_DATASERIES
|
||||||
|
{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
|
||||||
|
#endif
|
||||||
|
|
||||||
// End marker, don't touch.
|
// End marker, don't touch.
|
||||||
{ BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 }
|
{ BifEnum::Log::WRITER_DEFAULT, "None", 0, (WriterBackend* (*)(WriterFrontend* frontend))0 }
|
||||||
|
|
@ -51,7 +57,7 @@ struct Manager::Filter {
|
||||||
Func* postprocessor;
|
Func* postprocessor;
|
||||||
|
|
||||||
int num_fields;
|
int num_fields;
|
||||||
Field** fields;
|
threading::Field** fields;
|
||||||
|
|
||||||
// Vector indexed by field number. Each element is a list of record
|
// Vector indexed by field number. Each element is a list of record
|
||||||
// indices defining a path leading to the value across potential
|
// indices defining a path leading to the value across potential
|
||||||
|
|
@ -119,6 +125,7 @@ Manager::Stream::~Stream()
|
||||||
|
|
||||||
Manager::Manager()
|
Manager::Manager()
|
||||||
{
|
{
|
||||||
|
rotations_pending = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Manager::~Manager()
|
Manager::~Manager()
|
||||||
|
|
@ -127,6 +134,16 @@ Manager::~Manager()
|
||||||
delete *s;
|
delete *s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
list<string> Manager::SupportedFormats()
|
||||||
|
{
|
||||||
|
list<string> formats;
|
||||||
|
|
||||||
|
for ( WriterDefinition* ld = log_writers; ld->type != BifEnum::Log::WRITER_DEFAULT; ++ld )
|
||||||
|
formats.push_back(ld->name);
|
||||||
|
|
||||||
|
return formats;
|
||||||
|
}
|
||||||
|
|
||||||
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
||||||
{
|
{
|
||||||
WriterDefinition* ld = log_writers;
|
WriterDefinition* ld = log_writers;
|
||||||
|
|
@ -135,7 +152,7 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
||||||
{
|
{
|
||||||
if ( ld->type == BifEnum::Log::WRITER_DEFAULT )
|
if ( ld->type == BifEnum::Log::WRITER_DEFAULT )
|
||||||
{
|
{
|
||||||
reporter->Error("unknow writer when creating writer");
|
reporter->Error("unknown writer type requested");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -159,10 +176,8 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
|
||||||
// function.
|
// function.
|
||||||
ld->factory = 0;
|
ld->factory = 0;
|
||||||
|
|
||||||
DBG_LOG(DBG_LOGGING, "failed to init writer class %s",
|
reporter->Error("initialization of writer %s failed", ld->name);
|
||||||
ld->name);
|
return 0;
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -449,7 +464,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
|
||||||
|
|
||||||
filter->indices.push_back(new_indices);
|
filter->indices.push_back(new_indices);
|
||||||
|
|
||||||
filter->fields = (Field**)
|
filter->fields = (threading::Field**)
|
||||||
realloc(filter->fields,
|
realloc(filter->fields,
|
||||||
sizeof(Field) * ++filter->num_fields);
|
sizeof(Field) * ++filter->num_fields);
|
||||||
|
|
||||||
|
|
@ -459,7 +474,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
Field* field = new Field();
|
threading::Field* field = new threading::Field();
|
||||||
field->name = new_path;
|
field->name = new_path;
|
||||||
field->type = t->Tag();
|
field->type = t->Tag();
|
||||||
|
|
||||||
|
|
@ -571,7 +586,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
|
||||||
|
|
||||||
for ( int i = 0; i < filter->num_fields; i++ )
|
for ( int i = 0; i < filter->num_fields; i++ )
|
||||||
{
|
{
|
||||||
Field* field = filter->fields[i];
|
threading::Field* field = filter->fields[i];
|
||||||
DBG_LOG(DBG_LOGGING, " field %10s: %s",
|
DBG_LOG(DBG_LOGGING, " field %10s: %s",
|
||||||
field->name.c_str(), type_name(field->type));
|
field->name.c_str(), type_name(field->type));
|
||||||
}
|
}
|
||||||
|
|
@ -743,10 +758,10 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
|
||||||
|
|
||||||
// Copy the fields for WriterFrontend::Init() as it
|
// Copy the fields for WriterFrontend::Init() as it
|
||||||
// will take ownership.
|
// will take ownership.
|
||||||
Field** arg_fields = new Field*[filter->num_fields];
|
threading::Field** arg_fields = new threading::Field*[filter->num_fields];
|
||||||
|
|
||||||
for ( int j = 0; j < filter->num_fields; ++j )
|
for ( int j = 0; j < filter->num_fields; ++j )
|
||||||
arg_fields[j] = new Field(*filter->fields[j]);
|
arg_fields[j] = new threading::Field(*filter->fields[j]);
|
||||||
|
|
||||||
writer = CreateWriter(stream->id, filter->writer,
|
writer = CreateWriter(stream->id, filter->writer,
|
||||||
path, filter->num_fields,
|
path, filter->num_fields,
|
||||||
|
|
@ -897,10 +912,10 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
|
||||||
return lval;
|
return lval;
|
||||||
}
|
}
|
||||||
|
|
||||||
Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
||||||
RecordVal* columns)
|
RecordVal* columns)
|
||||||
{
|
{
|
||||||
Value** vals = new Value*[filter->num_fields];
|
threading::Value** vals = new threading::Value*[filter->num_fields];
|
||||||
|
|
||||||
for ( int i = 0; i < filter->num_fields; ++i )
|
for ( int i = 0; i < filter->num_fields; ++i )
|
||||||
{
|
{
|
||||||
|
|
@ -919,7 +934,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
||||||
if ( ! val )
|
if ( ! val )
|
||||||
{
|
{
|
||||||
// Value, or any of its parents, is not set.
|
// Value, or any of its parents, is not set.
|
||||||
vals[i] = new Value(filter->fields[i]->type, false);
|
vals[i] = new threading::Value(filter->fields[i]->type, false);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -932,7 +947,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
|
||||||
}
|
}
|
||||||
|
|
||||||
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
|
WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
|
||||||
int num_fields, const Field* const* fields, bool local, bool remote)
|
int num_fields, const threading::Field* const* fields, bool local, bool remote)
|
||||||
{
|
{
|
||||||
Stream* stream = FindStream(id);
|
Stream* stream = FindStream(id);
|
||||||
|
|
||||||
|
|
@ -996,7 +1011,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
|
||||||
return writer_obj;
|
return writer_obj;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Manager::DeleteVals(int num_fields, Value** vals)
|
void Manager::DeleteVals(int num_fields, threading::Value** vals)
|
||||||
{
|
{
|
||||||
// Note this code is duplicated in WriterBackend::DeleteVals().
|
// Note this code is duplicated in WriterBackend::DeleteVals().
|
||||||
for ( int i = 0; i < num_fields; i++ )
|
for ( int i = 0; i < num_fields; i++ )
|
||||||
|
|
@ -1006,7 +1021,7 @@ void Manager::DeleteVals(int num_fields, Value** vals)
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields,
|
bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields,
|
||||||
Value** vals)
|
threading::Value** vals)
|
||||||
{
|
{
|
||||||
Stream* stream = FindStream(id);
|
Stream* stream = FindStream(id);
|
||||||
|
|
||||||
|
|
@ -1113,10 +1128,19 @@ bool Manager::Flush(EnumVal* id)
|
||||||
|
|
||||||
void Manager::Terminate()
|
void Manager::Terminate()
|
||||||
{
|
{
|
||||||
|
// Make sure we process all the pending rotations.
|
||||||
|
while ( rotations_pending )
|
||||||
|
{
|
||||||
|
thread_mgr->ForceProcessing(); // A blatant layering violation ...
|
||||||
|
usleep(1000);
|
||||||
|
}
|
||||||
|
|
||||||
for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
|
for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
|
||||||
{
|
{
|
||||||
if ( *s )
|
if ( ! *s )
|
||||||
Flush((*s)->id);
|
continue;
|
||||||
|
|
||||||
|
Flush((*s)->id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1219,11 +1243,19 @@ void Manager::Rotate(WriterInfo* winfo)
|
||||||
|
|
||||||
// Trigger the rotation.
|
// Trigger the rotation.
|
||||||
winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
|
winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
|
||||||
|
|
||||||
|
++rotations_pending;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
|
bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
|
||||||
double open, double close, bool terminating)
|
double open, double close, bool terminating)
|
||||||
{
|
{
|
||||||
|
--rotations_pending;
|
||||||
|
|
||||||
|
if ( ! writer )
|
||||||
|
// Writer didn't produce local output.
|
||||||
|
return true;
|
||||||
|
|
||||||
DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
|
DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
|
||||||
writer->Path().c_str(), network_time, new_name.c_str());
|
writer->Path().c_str(), network_time, new_name.c_str());
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,6 @@ class RotationTimer;
|
||||||
|
|
||||||
namespace logging {
|
namespace logging {
|
||||||
|
|
||||||
|
|
||||||
class WriterBackend;
|
class WriterBackend;
|
||||||
class WriterFrontend;
|
class WriterFrontend;
|
||||||
class RotationFinishedMessage;
|
class RotationFinishedMessage;
|
||||||
|
|
@ -145,6 +144,11 @@ public:
|
||||||
*/
|
*/
|
||||||
void Terminate();
|
void Terminate();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a list of supported output formats.
|
||||||
|
*/
|
||||||
|
static list<string> SupportedFormats();
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
friend class WriterFrontend;
|
friend class WriterFrontend;
|
||||||
friend class RotationFinishedMessage;
|
friend class RotationFinishedMessage;
|
||||||
|
|
@ -196,6 +200,7 @@ private:
|
||||||
WriterInfo* FindWriter(WriterFrontend* writer);
|
WriterInfo* FindWriter(WriterFrontend* writer);
|
||||||
|
|
||||||
vector<Stream *> streams; // Indexed by stream enum.
|
vector<Stream *> streams; // Indexed by stream enum.
|
||||||
|
int rotations_pending; // Number of rotations not yet finished.
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -223,17 +223,6 @@ bool WriterBackend::Flush()
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool WriterBackend::Finish()
|
|
||||||
{
|
|
||||||
if ( ! DoFlush() )
|
|
||||||
{
|
|
||||||
DisableFrontend();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool WriterBackend::DoHeartbeat(double network_time, double current_time)
|
bool WriterBackend::DoHeartbeat(double network_time, double current_time)
|
||||||
{
|
{
|
||||||
MsgThread::DoHeartbeat(network_time, current_time);
|
MsgThread::DoHeartbeat(network_time, current_time);
|
||||||
|
|
@ -279,4 +268,9 @@ string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string WriterBackend::Render(double d) const
|
||||||
|
{
|
||||||
|
char buf[256];
|
||||||
|
modp_dtoa(d, buf, 6);
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -101,15 +101,6 @@ public:
|
||||||
*/
|
*/
|
||||||
bool Rotate(string rotated_path, double open, double close, bool terminating);
|
bool Rotate(string rotated_path, double open, double close, bool terminating);
|
||||||
|
|
||||||
/**
|
|
||||||
* Finishes writing to this logger in a regularl fashion. Must not be
|
|
||||||
* called if an error has been indicated earlier. After calling this,
|
|
||||||
* no further writing must be performed.
|
|
||||||
*
|
|
||||||
* @return False if an error occured.
|
|
||||||
*/
|
|
||||||
bool Finish();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Disables the frontend that has instantiated this backend. Once
|
* Disables the frontend that has instantiated this backend. Once
|
||||||
* disabled,the frontend will not send any further message over.
|
* disabled,the frontend will not send any further message over.
|
||||||
|
|
@ -174,7 +165,17 @@ public:
|
||||||
*/
|
*/
|
||||||
string Render(const threading::Value::subnet_t& subnet) const;
|
string Render(const threading::Value::subnet_t& subnet) const;
|
||||||
|
|
||||||
|
/** Helper method to render a double in Bro's standard precision.
|
||||||
|
*
|
||||||
|
* @param d The double.
|
||||||
|
*
|
||||||
|
* @return An ASCII representation of the double.
|
||||||
|
*/
|
||||||
|
string Render(double d) const;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
friend class FinishMessage;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writer-specific intialization method.
|
* Writer-specific intialization method.
|
||||||
*
|
*
|
||||||
|
|
@ -272,26 +273,18 @@ protected:
|
||||||
bool terminating) = 0;
|
bool terminating) = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writer-specific method implementing log output finalization at
|
* Writer-specific method called just before the threading system is
|
||||||
* termination. Not called when any of the other methods has
|
* going to shutdown.
|
||||||
* previously signaled an error, i.e., executing this method signals
|
|
||||||
* a regular shutdown of the writer.
|
|
||||||
*
|
*
|
||||||
* A writer implementation must override this method but it can just
|
* This method can be overridden but one must call
|
||||||
* ignore calls if flushing doesn't align with its semantics.
|
* WriterBackend::DoFinish().
|
||||||
*
|
|
||||||
* If the method returns false, it will be assumed that a fatal error
|
|
||||||
* has occured that prevents the writer from further operation; it
|
|
||||||
* will then be disabled and eventually deleted. When returning
|
|
||||||
* false, an implementation should also call Error() to indicate what
|
|
||||||
* happened.
|
|
||||||
*/
|
*/
|
||||||
virtual bool DoFinish() = 0;
|
virtual bool DoFinish() { return MsgThread::DoFinish(); }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Triggered by regular heartbeat messages from the main thread.
|
* Triggered by regular heartbeat messages from the main thread.
|
||||||
*
|
*
|
||||||
* This method can be overridden but once must call
|
* This method can be overridden but one must call
|
||||||
* WriterBackend::DoHeartbeat().
|
* WriterBackend::DoHeartbeat().
|
||||||
*/
|
*/
|
||||||
virtual bool DoHeartbeat(double network_time, double current_time);
|
virtual bool DoHeartbeat(double network_time, double current_time);
|
||||||
|
|
|
||||||
|
|
@ -90,7 +90,7 @@ public:
|
||||||
FinishMessage(WriterBackend* backend)
|
FinishMessage(WriterBackend* backend)
|
||||||
: threading::InputMessage<WriterBackend>("Finish", backend) {}
|
: threading::InputMessage<WriterBackend>("Finish", backend) {}
|
||||||
|
|
||||||
virtual bool Process() { return Object()->Finish(); }
|
virtual bool Process() { return Object()->DoFinish(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
@ -117,8 +117,9 @@ WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool ar
|
||||||
if ( local )
|
if ( local )
|
||||||
{
|
{
|
||||||
backend = log_mgr->CreateBackend(this, writer->AsEnum());
|
backend = log_mgr->CreateBackend(this, writer->AsEnum());
|
||||||
assert(backend);
|
|
||||||
backend->Start();
|
if ( backend )
|
||||||
|
backend->Start();
|
||||||
}
|
}
|
||||||
|
|
||||||
else
|
else
|
||||||
|
|
@ -256,6 +257,10 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool
|
||||||
|
|
||||||
if ( backend )
|
if ( backend )
|
||||||
backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
|
backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
|
||||||
|
else
|
||||||
|
// Still signal log manager that we're done, but signal that
|
||||||
|
// nothing happened by setting the writer to zeri.
|
||||||
|
log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating);
|
||||||
}
|
}
|
||||||
|
|
||||||
void WriterFrontend::Finish()
|
void WriterFrontend::Finish()
|
||||||
|
|
|
||||||
|
|
@ -69,8 +69,7 @@ bool Ascii::WriteHeaderField(const string& key, const string& val)
|
||||||
return (fwrite(str.c_str(), str.length(), 1, file) == 1);
|
return (fwrite(str.c_str(), str.length(), 1, file) == 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ascii::DoInit(string path, int num_fields,
|
bool Ascii::DoInit(string path, int num_fields, const Field* const * fields)
|
||||||
const Field* const * fields)
|
|
||||||
{
|
{
|
||||||
if ( output_to_stdout )
|
if ( output_to_stdout )
|
||||||
path = "/dev/stdout";
|
path = "/dev/stdout";
|
||||||
|
|
@ -87,6 +86,9 @@ bool Ascii::DoInit(string path, int num_fields,
|
||||||
|
|
||||||
if ( include_header )
|
if ( include_header )
|
||||||
{
|
{
|
||||||
|
string names;
|
||||||
|
string types;
|
||||||
|
|
||||||
string str = string(header_prefix, header_prefix_len)
|
string str = string(header_prefix, header_prefix_len)
|
||||||
+ "separator " // Always use space as separator here.
|
+ "separator " // Always use space as separator here.
|
||||||
+ get_escaped_string(string(separator, separator_len), false)
|
+ get_escaped_string(string(separator, separator_len), false)
|
||||||
|
|
@ -104,9 +106,6 @@ bool Ascii::DoInit(string path, int num_fields,
|
||||||
WriteHeaderField("path", get_escaped_string(path, false))) )
|
WriteHeaderField("path", get_escaped_string(path, false))) )
|
||||||
goto write_error;
|
goto write_error;
|
||||||
|
|
||||||
string names;
|
|
||||||
string types;
|
|
||||||
|
|
||||||
for ( int i = 0; i < num_fields; ++i )
|
for ( int i = 0; i < num_fields; ++i )
|
||||||
{
|
{
|
||||||
if ( i > 0 )
|
if ( i > 0 )
|
||||||
|
|
@ -115,15 +114,8 @@ bool Ascii::DoInit(string path, int num_fields,
|
||||||
types += string(separator, separator_len);
|
types += string(separator, separator_len);
|
||||||
}
|
}
|
||||||
|
|
||||||
const Field* field = fields[i];
|
names += fields[i]->name;
|
||||||
names += field->name;
|
types += fields[i]->TypeName();
|
||||||
types += type_name(field->type);
|
|
||||||
if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) )
|
|
||||||
{
|
|
||||||
types += "[";
|
|
||||||
types += type_name(field->subtype);
|
|
||||||
types += "]";
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( ! (WriteHeaderField("fields", names)
|
if ( ! (WriteHeaderField("fields", names)
|
||||||
|
|
@ -146,7 +138,7 @@ bool Ascii::DoFlush()
|
||||||
|
|
||||||
bool Ascii::DoFinish()
|
bool Ascii::DoFinish()
|
||||||
{
|
{
|
||||||
return true;
|
return WriterBackend::DoFinish();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
|
bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
|
||||||
|
|
@ -184,15 +176,19 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
|
||||||
desc->Add(Render(val->val.addr_val));
|
desc->Add(Render(val->val.addr_val));
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case TYPE_TIME:
|
case TYPE_DOUBLE:
|
||||||
case TYPE_INTERVAL:
|
// Rendering via Add() truncates trailing 0s after the
|
||||||
char buf[256];
|
// decimal point. The difference with TIME/INTERVAL is mainly
|
||||||
modp_dtoa(val->val.double_val, buf, 6);
|
// to keep the log format consistent.
|
||||||
desc->Add(buf);
|
desc->Add(val->val.double_val);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case TYPE_DOUBLE:
|
case TYPE_INTERVAL:
|
||||||
desc->Add(val->val.double_val);
|
case TYPE_TIME:
|
||||||
|
// Rendering via Render() keeps trailing 0s after the decimal
|
||||||
|
// point. The difference with DOUBLEis mainly to keep the log
|
||||||
|
// format consistent.
|
||||||
|
desc->Add(Render(val->val.double_val));
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case TYPE_ENUM:
|
case TYPE_ENUM:
|
||||||
|
|
|
||||||
417
src/logging/writers/DataSeries.cc
Normal file
417
src/logging/writers/DataSeries.cc
Normal file
|
|
@ -0,0 +1,417 @@
|
||||||
|
// See the file "COPYING" in the main distribution directory for copyright.
|
||||||
|
|
||||||
|
#include <map>
|
||||||
|
#include <string>
|
||||||
|
#include <errno.h>
|
||||||
|
|
||||||
|
#include <DataSeries/GeneralField.hpp>
|
||||||
|
|
||||||
|
#include "NetVar.h"
|
||||||
|
#include "threading/SerialTypes.h"
|
||||||
|
|
||||||
|
#include "DataSeries.h"
|
||||||
|
|
||||||
|
using namespace logging;
|
||||||
|
using namespace writer;
|
||||||
|
|
||||||
|
std::string DataSeries::LogValueToString(threading::Value *val)
|
||||||
|
{
|
||||||
|
// In some cases, no value is attached. If this is the case, return
|
||||||
|
// an empty string.
|
||||||
|
if( ! val->present )
|
||||||
|
return "";
|
||||||
|
|
||||||
|
switch(val->type) {
|
||||||
|
case TYPE_BOOL:
|
||||||
|
return (val->val.int_val ? "true" : "false");
|
||||||
|
|
||||||
|
case TYPE_INT:
|
||||||
|
{
|
||||||
|
std::ostringstream ostr;
|
||||||
|
ostr << val->val.int_val;
|
||||||
|
return ostr.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
case TYPE_COUNT:
|
||||||
|
case TYPE_COUNTER:
|
||||||
|
case TYPE_PORT:
|
||||||
|
{
|
||||||
|
std::ostringstream ostr;
|
||||||
|
ostr << val->val.uint_val;
|
||||||
|
return ostr.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
case TYPE_SUBNET:
|
||||||
|
return Render(val->val.subnet_val);
|
||||||
|
|
||||||
|
case TYPE_ADDR:
|
||||||
|
return Render(val->val.addr_val);
|
||||||
|
|
||||||
|
// Note: These two cases are relatively special. We need to convert
|
||||||
|
// these values into their integer equivalents to maximize precision.
|
||||||
|
// At the moment, there won't be a noticeable effect (Bro uses the
|
||||||
|
// double format everywhere internally, so we've already lost the
|
||||||
|
// precision we'd gain here), but timestamps may eventually switch to
|
||||||
|
// this representation within Bro.
|
||||||
|
//
|
||||||
|
// In the near-term, this *should* lead to better pack_relative (and
|
||||||
|
// thus smaller output files).
|
||||||
|
case TYPE_TIME:
|
||||||
|
case TYPE_INTERVAL:
|
||||||
|
if ( ds_use_integer_for_time )
|
||||||
|
{
|
||||||
|
std::ostringstream ostr;
|
||||||
|
ostr << (uint64_t)(DataSeries::TIME_SCALE * val->val.double_val);
|
||||||
|
return ostr.str();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
return Render(val->val.double_val);
|
||||||
|
|
||||||
|
case TYPE_DOUBLE:
|
||||||
|
return Render(val->val.double_val);
|
||||||
|
|
||||||
|
case TYPE_ENUM:
|
||||||
|
case TYPE_STRING:
|
||||||
|
case TYPE_FILE:
|
||||||
|
case TYPE_FUNC:
|
||||||
|
if ( ! val->val.string_val->size() )
|
||||||
|
return "";
|
||||||
|
|
||||||
|
return string(val->val.string_val->data(), val->val.string_val->size());
|
||||||
|
|
||||||
|
case TYPE_TABLE:
|
||||||
|
{
|
||||||
|
if ( ! val->val.set_val.size )
|
||||||
|
return "";
|
||||||
|
|
||||||
|
string tmpString = "";
|
||||||
|
|
||||||
|
for ( int j = 0; j < val->val.set_val.size; j++ )
|
||||||
|
{
|
||||||
|
if ( j > 0 )
|
||||||
|
tmpString += ds_set_separator;
|
||||||
|
|
||||||
|
tmpString += LogValueToString(val->val.set_val.vals[j]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return tmpString;
|
||||||
|
}
|
||||||
|
|
||||||
|
case TYPE_VECTOR:
|
||||||
|
{
|
||||||
|
if ( ! val->val.vector_val.size )
|
||||||
|
return "";
|
||||||
|
|
||||||
|
string tmpString = "";
|
||||||
|
|
||||||
|
for ( int j = 0; j < val->val.vector_val.size; j++ )
|
||||||
|
{
|
||||||
|
if ( j > 0 )
|
||||||
|
tmpString += ds_set_separator;
|
||||||
|
|
||||||
|
tmpString += LogValueToString(val->val.vector_val.vals[j]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return tmpString;
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
InternalError(Fmt("unknown type %s in DataSeries::LogValueToString", type_name(val->type)));
|
||||||
|
return "cannot be reached";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
string DataSeries::GetDSFieldType(const threading::Field *field)
|
||||||
|
{
|
||||||
|
switch(field->type) {
|
||||||
|
case TYPE_BOOL:
|
||||||
|
return "bool";
|
||||||
|
|
||||||
|
case TYPE_COUNT:
|
||||||
|
case TYPE_COUNTER:
|
||||||
|
case TYPE_PORT:
|
||||||
|
case TYPE_INT:
|
||||||
|
return "int64";
|
||||||
|
|
||||||
|
case TYPE_DOUBLE:
|
||||||
|
return "double";
|
||||||
|
|
||||||
|
case TYPE_TIME:
|
||||||
|
case TYPE_INTERVAL:
|
||||||
|
return ds_use_integer_for_time ? "int64" : "double";
|
||||||
|
|
||||||
|
case TYPE_SUBNET:
|
||||||
|
case TYPE_ADDR:
|
||||||
|
case TYPE_ENUM:
|
||||||
|
case TYPE_STRING:
|
||||||
|
case TYPE_FILE:
|
||||||
|
case TYPE_TABLE:
|
||||||
|
case TYPE_VECTOR:
|
||||||
|
case TYPE_FUNC:
|
||||||
|
return "variable32";
|
||||||
|
|
||||||
|
default:
|
||||||
|
InternalError(Fmt("unknown type %s in DataSeries::GetDSFieldType", type_name(field->type)));
|
||||||
|
return "cannot be reached";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
string DataSeries::BuildDSSchemaFromFieldTypes(const vector<SchemaValue>& vals, string sTitle)
|
||||||
|
{
|
||||||
|
if( ! sTitle.size() )
|
||||||
|
sTitle = "GenericBroStream";
|
||||||
|
|
||||||
|
string xmlschema = "<ExtentType name=\""
|
||||||
|
+ sTitle
|
||||||
|
+ "\" version=\"1.0\" namespace=\"bro-ids.org\">\n";
|
||||||
|
|
||||||
|
for( size_t i = 0; i < vals.size(); ++i )
|
||||||
|
{
|
||||||
|
xmlschema += "\t<field type=\""
|
||||||
|
+ vals[i].ds_type
|
||||||
|
+ "\" name=\""
|
||||||
|
+ vals[i].field_name
|
||||||
|
+ "\" " + vals[i].field_options
|
||||||
|
+ "/>\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
xmlschema += "</ExtentType>\n";
|
||||||
|
|
||||||
|
for( size_t i = 0; i < vals.size(); ++i )
|
||||||
|
{
|
||||||
|
xmlschema += "<!-- " + vals[i].field_name
|
||||||
|
+ " : " + vals[i].bro_type
|
||||||
|
+ " -->\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
return xmlschema;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
|
||||||
|
{
|
||||||
|
switch( field->type ) {
|
||||||
|
case TYPE_TIME:
|
||||||
|
case TYPE_INTERVAL:
|
||||||
|
{
|
||||||
|
std::string s;
|
||||||
|
s += "pack_relative=\"" + std::string(field->name) + "\"";
|
||||||
|
|
||||||
|
if ( ! ds_use_integer_for_time )
|
||||||
|
s += " pack_scale=\"1e-6\" print_format=\"%.6f\" pack_scale_warn=\"no\"";
|
||||||
|
else
|
||||||
|
s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
|
||||||
|
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
case TYPE_SUBNET:
|
||||||
|
case TYPE_ADDR:
|
||||||
|
case TYPE_ENUM:
|
||||||
|
case TYPE_STRING:
|
||||||
|
case TYPE_FILE:
|
||||||
|
case TYPE_TABLE:
|
||||||
|
case TYPE_VECTOR:
|
||||||
|
return "pack_unique=\"yes\"";
|
||||||
|
|
||||||
|
default:
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DataSeries::DataSeries(WriterFrontend* frontend) : WriterBackend(frontend)
|
||||||
|
{
|
||||||
|
ds_compression = string((const char *)BifConst::LogDataSeries::compression->Bytes(),
|
||||||
|
BifConst::LogDataSeries::compression->Len());
|
||||||
|
ds_dump_schema = BifConst::LogDataSeries::dump_schema;
|
||||||
|
ds_extent_size = BifConst::LogDataSeries::extent_size;
|
||||||
|
ds_num_threads = BifConst::LogDataSeries::num_threads;
|
||||||
|
ds_use_integer_for_time = BifConst::LogDataSeries::use_integer_for_time;
|
||||||
|
ds_set_separator = ",";
|
||||||
|
}
|
||||||
|
|
||||||
|
DataSeries::~DataSeries()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::OpenLog(string path)
|
||||||
|
{
|
||||||
|
log_file = new DataSeriesSink(path + ".ds", compress_type);
|
||||||
|
log_file->writeExtentLibrary(log_types);
|
||||||
|
|
||||||
|
for( size_t i = 0; i < schema_list.size(); ++i )
|
||||||
|
extents.insert(std::make_pair(schema_list[i].field_name,
|
||||||
|
GeneralField::create(log_series, schema_list[i].field_name)));
|
||||||
|
|
||||||
|
if ( ds_extent_size < ROW_MIN )
|
||||||
|
{
|
||||||
|
Warning(Fmt("%d is not a valid value for 'rows'. Using min of %d instead", (int)ds_extent_size, (int)ROW_MIN));
|
||||||
|
ds_extent_size = ROW_MIN;
|
||||||
|
}
|
||||||
|
|
||||||
|
else if( ds_extent_size > ROW_MAX )
|
||||||
|
{
|
||||||
|
Warning(Fmt("%d is not a valid value for 'rows'. Using max of %d instead", (int)ds_extent_size, (int)ROW_MAX));
|
||||||
|
ds_extent_size = ROW_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
log_output = new OutputModule(*log_file, log_series, log_type, ds_extent_size);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::DoInit(string path, int num_fields, const threading::Field* const * fields)
|
||||||
|
{
|
||||||
|
// We first construct an XML schema thing (and, if ds_dump_schema is
|
||||||
|
// set, dump it to path + ".ds.xml"). Assuming that goes well, we
|
||||||
|
// use that schema to build our output logfile and prepare it to be
|
||||||
|
// written to.
|
||||||
|
|
||||||
|
// Note: compressor count must be set *BEFORE* DataSeriesSink is
|
||||||
|
// instantiated.
|
||||||
|
if( ds_num_threads < THREAD_MIN && ds_num_threads != 0 )
|
||||||
|
{
|
||||||
|
Warning(Fmt("%d is too few threads! Using %d instead", (int)ds_num_threads, (int)THREAD_MIN));
|
||||||
|
ds_num_threads = THREAD_MIN;
|
||||||
|
}
|
||||||
|
|
||||||
|
if( ds_num_threads > THREAD_MAX )
|
||||||
|
{
|
||||||
|
Warning(Fmt("%d is too many threads! Dropping back to %d", (int)ds_num_threads, (int)THREAD_MAX));
|
||||||
|
ds_num_threads = THREAD_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
if( ds_num_threads > 0 )
|
||||||
|
DataSeriesSink::setCompressorCount(ds_num_threads);
|
||||||
|
|
||||||
|
for ( int i = 0; i < num_fields; i++ )
|
||||||
|
{
|
||||||
|
const threading::Field* field = fields[i];
|
||||||
|
SchemaValue val;
|
||||||
|
val.ds_type = GetDSFieldType(field);
|
||||||
|
val.field_name = string(field->name);
|
||||||
|
val.field_options = GetDSOptionsForType(field);
|
||||||
|
val.bro_type = field->TypeName();
|
||||||
|
schema_list.push_back(val);
|
||||||
|
}
|
||||||
|
|
||||||
|
string schema = BuildDSSchemaFromFieldTypes(schema_list, path);
|
||||||
|
|
||||||
|
if( ds_dump_schema )
|
||||||
|
{
|
||||||
|
FILE* pFile = fopen ( string(path + ".ds.xml").c_str() , "wb" );
|
||||||
|
|
||||||
|
if( pFile )
|
||||||
|
{
|
||||||
|
fwrite(schema.c_str(), 1, schema.length(), pFile);
|
||||||
|
fclose(pFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
else
|
||||||
|
Error(Fmt("cannot dump schema: %s", strerror(errno)));
|
||||||
|
}
|
||||||
|
|
||||||
|
compress_type = Extent::compress_all;
|
||||||
|
|
||||||
|
if( ds_compression == "lzf" )
|
||||||
|
compress_type = Extent::compress_lzf;
|
||||||
|
|
||||||
|
else if( ds_compression == "lzo" )
|
||||||
|
compress_type = Extent::compress_lzo;
|
||||||
|
|
||||||
|
else if( ds_compression == "gz" )
|
||||||
|
compress_type = Extent::compress_gz;
|
||||||
|
|
||||||
|
else if( ds_compression == "bz2" )
|
||||||
|
compress_type = Extent::compress_bz2;
|
||||||
|
|
||||||
|
else if( ds_compression == "none" )
|
||||||
|
compress_type = Extent::compress_none;
|
||||||
|
|
||||||
|
else if( ds_compression == "any" )
|
||||||
|
compress_type = Extent::compress_all;
|
||||||
|
|
||||||
|
else
|
||||||
|
Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str()));
|
||||||
|
|
||||||
|
log_type = log_types.registerTypePtr(schema);
|
||||||
|
log_series.setType(log_type);
|
||||||
|
|
||||||
|
return OpenLog(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::DoFlush()
|
||||||
|
{
|
||||||
|
// Flushing is handled by DataSeries automatically, so this function
|
||||||
|
// doesn't do anything.
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void DataSeries::CloseLog()
|
||||||
|
{
|
||||||
|
for( ExtentIterator iter = extents.begin(); iter != extents.end(); ++iter )
|
||||||
|
delete iter->second;
|
||||||
|
|
||||||
|
extents.clear();
|
||||||
|
|
||||||
|
// Don't delete the file before you delete the output, or bad things
|
||||||
|
// will happen.
|
||||||
|
delete log_output;
|
||||||
|
delete log_file;
|
||||||
|
|
||||||
|
log_output = 0;
|
||||||
|
log_file = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::DoFinish()
|
||||||
|
{
|
||||||
|
CloseLog();
|
||||||
|
|
||||||
|
return WriterBackend::DoFinish();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
|
||||||
|
threading::Value** vals)
|
||||||
|
{
|
||||||
|
log_output->newRecord();
|
||||||
|
|
||||||
|
for( size_t i = 0; i < (size_t)num_fields; ++i )
|
||||||
|
{
|
||||||
|
ExtentIterator iter = extents.find(fields[i]->name);
|
||||||
|
assert(iter != extents.end());
|
||||||
|
|
||||||
|
if( iter != extents.end() )
|
||||||
|
{
|
||||||
|
GeneralField *cField = iter->second;
|
||||||
|
|
||||||
|
if( vals[i]->present )
|
||||||
|
cField->set(LogValueToString(vals[i]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::DoRotate(string rotated_path, double open, double close, bool terminating)
|
||||||
|
{
|
||||||
|
// Note that if DS files are rotated too often, the aggregate log
|
||||||
|
// size will be (much) larger.
|
||||||
|
CloseLog();
|
||||||
|
|
||||||
|
string dsname = Path() + ".ds";
|
||||||
|
string nname = rotated_path + ".ds";
|
||||||
|
rename(dsname.c_str(), nname.c_str());
|
||||||
|
|
||||||
|
if ( ! FinishedRotation(nname, dsname, open, close, terminating) )
|
||||||
|
{
|
||||||
|
Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return OpenLog(Path());
|
||||||
|
}
|
||||||
|
|
||||||
|
bool DataSeries::DoSetBuf(bool enabled)
|
||||||
|
{
|
||||||
|
// DataSeries is *always* buffered to some degree. This option is ignored.
|
||||||
|
return true;
|
||||||
|
}
|
||||||
124
src/logging/writers/DataSeries.h
Normal file
124
src/logging/writers/DataSeries.h
Normal file
|
|
@ -0,0 +1,124 @@
|
||||||
|
// See the file "COPYING" in the main distribution directory for copyright.
|
||||||
|
//
|
||||||
|
// A binary log writer producing DataSeries output. See doc/data-series.rst
|
||||||
|
// for more information.
|
||||||
|
|
||||||
|
#ifndef LOGGING_WRITER_DATA_SERIES_H
|
||||||
|
#define LOGGING_WRITER_DATA_SERIES_H
|
||||||
|
|
||||||
|
#include <DataSeries/ExtentType.hpp>
|
||||||
|
#include <DataSeries/DataSeriesFile.hpp>
|
||||||
|
#include <DataSeries/DataSeriesModule.hpp>
|
||||||
|
#include <DataSeries/GeneralField.hpp>
|
||||||
|
|
||||||
|
#include "../WriterBackend.h"
|
||||||
|
|
||||||
|
namespace logging { namespace writer {
|
||||||
|
|
||||||
|
class DataSeries : public WriterBackend {
|
||||||
|
public:
|
||||||
|
DataSeries(WriterFrontend* frontend);
|
||||||
|
~DataSeries();
|
||||||
|
|
||||||
|
static WriterBackend* Instantiate(WriterFrontend* frontend)
|
||||||
|
{ return new DataSeries(frontend); }
|
||||||
|
|
||||||
|
protected:
|
||||||
|
// Overidden from WriterBackend.
|
||||||
|
|
||||||
|
virtual bool DoInit(string path, int num_fields,
|
||||||
|
const threading::Field* const * fields);
|
||||||
|
|
||||||
|
virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
|
||||||
|
threading::Value** vals);
|
||||||
|
virtual bool DoSetBuf(bool enabled);
|
||||||
|
virtual bool DoRotate(string rotated_path, double open,
|
||||||
|
double close, bool terminating);
|
||||||
|
virtual bool DoFlush();
|
||||||
|
virtual bool DoFinish();
|
||||||
|
|
||||||
|
private:
|
||||||
|
static const size_t ROW_MIN = 2048; // Minimum extent size.
|
||||||
|
static const size_t ROW_MAX = (1024 * 1024 * 100); // Maximum extent size.
|
||||||
|
static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn.
|
||||||
|
static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn.
|
||||||
|
static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers.
|
||||||
|
const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE.
|
||||||
|
|
||||||
|
struct SchemaValue
|
||||||
|
{
|
||||||
|
string ds_type;
|
||||||
|
string bro_type;
|
||||||
|
string field_name;
|
||||||
|
string field_options;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turns a log value into a std::string. Uses an ostringstream to do the
|
||||||
|
* heavy lifting, but still need to switch on the type to know which value
|
||||||
|
* in the union to give to the string string for processing.
|
||||||
|
*
|
||||||
|
* @param val The value we wish to convert to a string
|
||||||
|
* @return the string value of val
|
||||||
|
*/
|
||||||
|
std::string LogValueToString(threading::Value *val);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes a field type and converts it to a relevant DataSeries type.
|
||||||
|
*
|
||||||
|
* @param field We extract the type from this and convert it into a relevant DS type.
|
||||||
|
* @return String representation of type that DataSeries can understand.
|
||||||
|
*/
|
||||||
|
string GetDSFieldType(const threading::Field *field);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Are there any options we should put into the XML schema?
|
||||||
|
*
|
||||||
|
* @param field We extract the type from this and return any options that make sense for that type.
|
||||||
|
* @return Options that can be added directly to the XML (e.g. "pack_relative=\"yes\"")
|
||||||
|
*/
|
||||||
|
std::string GetDSOptionsForType(const threading::Field *field);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes a list of types, a list of names, and a title, and uses it to construct a valid DataSeries XML schema
|
||||||
|
* thing, which is then returned as a std::string
|
||||||
|
*
|
||||||
|
* @param opts std::vector of strings containing a list of options to be appended to each field (e.g. "pack_relative=yes")
|
||||||
|
* @param sTitle Name of this schema. Ideally, these schemas would be aggregated and re-used.
|
||||||
|
*/
|
||||||
|
string BuildDSSchemaFromFieldTypes(const vector<SchemaValue>& vals, string sTitle);
|
||||||
|
|
||||||
|
/** Closes the currently open file. */
|
||||||
|
void CloseLog();
|
||||||
|
|
||||||
|
/** Opens a new file. */
|
||||||
|
bool OpenLog(string path);
|
||||||
|
|
||||||
|
typedef std::map<string, GeneralField *> ExtentMap;
|
||||||
|
typedef ExtentMap::iterator ExtentIterator;
|
||||||
|
|
||||||
|
// Internal DataSeries structures we need to keep track of.
|
||||||
|
vector<SchemaValue> schema_list;
|
||||||
|
ExtentTypeLibrary log_types;
|
||||||
|
ExtentType::Ptr log_type;
|
||||||
|
ExtentSeries log_series;
|
||||||
|
ExtentMap extents;
|
||||||
|
int compress_type;
|
||||||
|
|
||||||
|
DataSeriesSink* log_file;
|
||||||
|
OutputModule* log_output;
|
||||||
|
|
||||||
|
// Options set from the script-level.
|
||||||
|
uint64 ds_extent_size;
|
||||||
|
uint64 ds_num_threads;
|
||||||
|
string ds_compression;
|
||||||
|
bool ds_dump_schema;
|
||||||
|
bool ds_use_integer_for_time;
|
||||||
|
string ds_set_separator;
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
21
src/main.cc
21
src/main.cc
|
|
@ -203,6 +203,27 @@ void usage()
|
||||||
fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str());
|
fprintf(stderr, " $BRO_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str());
|
||||||
fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n");
|
fprintf(stderr, " $BRO_PROFILER_FILE | Output file for script execution statistics (not set)\n");
|
||||||
|
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
fprintf(stderr, " Supported log formats: ");
|
||||||
|
|
||||||
|
bool first = true;
|
||||||
|
list<string> fmts = logging::Manager::SupportedFormats();
|
||||||
|
|
||||||
|
for ( list<string>::const_iterator i = fmts.begin(); i != fmts.end(); ++i )
|
||||||
|
{
|
||||||
|
if ( *i == "None" )
|
||||||
|
// Skip, it's uninteresting.
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if ( ! first )
|
||||||
|
fprintf(stderr, ",");
|
||||||
|
|
||||||
|
fprintf(stderr, "%s", (*i).c_str());
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,12 @@ public:
|
||||||
*/
|
*/
|
||||||
int NumThreads() const { return all_threads.size(); }
|
int NumThreads() const { return all_threads.size(); }
|
||||||
|
|
||||||
|
/** Manually triggers processing of any thread input. This can be useful
|
||||||
|
* if the main thread is waiting for a specific message from a child.
|
||||||
|
* Usually, though, one should avoid using it.
|
||||||
|
*/
|
||||||
|
void ForceProcessing() { Process(); }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
friend class BasicThread;
|
friend class BasicThread;
|
||||||
friend class MsgThread;
|
friend class MsgThread;
|
||||||
|
|
|
||||||
|
|
@ -10,13 +10,21 @@ namespace threading {
|
||||||
|
|
||||||
////// Messages.
|
////// Messages.
|
||||||
|
|
||||||
// Signals child thread to terminate. This is actually a no-op; its only
|
// Signals child thread to shutdown operation.
|
||||||
// purpose is unblock the current read operation so that the child's Run()
|
class FinishMessage : public InputMessage<MsgThread>
|
||||||
// methods can check the termination status.
|
|
||||||
class TerminateMessage : public InputMessage<MsgThread>
|
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
TerminateMessage(MsgThread* thread) : InputMessage<MsgThread>("Terminate", thread) { }
|
FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread) { }
|
||||||
|
|
||||||
|
virtual bool Process() { return Object()->DoFinish(); }
|
||||||
|
};
|
||||||
|
|
||||||
|
// A dummy message that's only purpose is unblock the current read operation
|
||||||
|
// so that the child's Run() methods can check the termination status.
|
||||||
|
class UnblockMessage : public InputMessage<MsgThread>
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread) { }
|
||||||
|
|
||||||
virtual bool Process() { return true; }
|
virtual bool Process() { return true; }
|
||||||
};
|
};
|
||||||
|
|
@ -130,13 +138,29 @@ bool ReporterMessage::Process()
|
||||||
MsgThread::MsgThread() : BasicThread()
|
MsgThread::MsgThread() : BasicThread()
|
||||||
{
|
{
|
||||||
cnt_sent_in = cnt_sent_out = 0;
|
cnt_sent_in = cnt_sent_out = 0;
|
||||||
|
finished = false;
|
||||||
thread_mgr->AddMsgThread(this);
|
thread_mgr->AddMsgThread(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
void MsgThread::OnStop()
|
void MsgThread::OnStop()
|
||||||
{
|
{
|
||||||
// This is to unblock the current queue read operation.
|
// Signal thread to terminate and wait until it has acknowledged.
|
||||||
SendIn(new TerminateMessage(this), true);
|
SendIn(new FinishMessage(this), true);
|
||||||
|
|
||||||
|
int cnt = 0;
|
||||||
|
while ( ! finished )
|
||||||
|
{
|
||||||
|
if ( ++cnt > 1000 ) // Insurance against broken threads ...
|
||||||
|
{
|
||||||
|
reporter->Warning("thread %s didn't finish in time", Name().c_str());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
usleep(1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
// One more message to make sure the current queue read operation unblocks.
|
||||||
|
SendIn(new UnblockMessage(this), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
void MsgThread::Heartbeat()
|
void MsgThread::Heartbeat()
|
||||||
|
|
@ -157,6 +181,14 @@ bool MsgThread::DoHeartbeat(double network_time, double current_time)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool MsgThread::DoFinish()
|
||||||
|
{
|
||||||
|
// This is thread-safe "enough", we're the only one ever writing
|
||||||
|
// there.
|
||||||
|
finished = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
void MsgThread::Info(const char* msg)
|
void MsgThread::Info(const char* msg)
|
||||||
{
|
{
|
||||||
SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg));
|
SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg));
|
||||||
|
|
@ -189,7 +221,9 @@ void MsgThread::InternalWarning(const char* msg)
|
||||||
|
|
||||||
void MsgThread::InternalError(const char* msg)
|
void MsgThread::InternalError(const char* msg)
|
||||||
{
|
{
|
||||||
SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg));
|
// This one aborts immediately.
|
||||||
|
fprintf(stderr, "internal error in thread: %s\n", msg);
|
||||||
|
abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef DEBUG
|
#ifdef DEBUG
|
||||||
|
|
|
||||||
|
|
@ -171,6 +171,8 @@ public:
|
||||||
protected:
|
protected:
|
||||||
friend class Manager;
|
friend class Manager;
|
||||||
friend class HeartbeatMessage;
|
friend class HeartbeatMessage;
|
||||||
|
friend class FinishMessage;
|
||||||
|
friend class FinishedMessage;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Pops a message sent by the child from the child-to-main queue.
|
* Pops a message sent by the child from the child-to-main queue.
|
||||||
|
|
@ -215,6 +217,12 @@ protected:
|
||||||
*/
|
*/
|
||||||
virtual bool DoHeartbeat(double network_time, double current_time);
|
virtual bool DoHeartbeat(double network_time, double current_time);
|
||||||
|
|
||||||
|
/** Triggered for execution in the child thread just before shutting threads down.
|
||||||
|
* The child thread should finish its operations and then *must*
|
||||||
|
* call this class' implementation.
|
||||||
|
*/
|
||||||
|
virtual bool DoFinish();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
/**
|
/**
|
||||||
* Pops a message sent by the main thread from the main-to-chold
|
* Pops a message sent by the main thread from the main-to-chold
|
||||||
|
|
@ -270,6 +278,8 @@ private:
|
||||||
|
|
||||||
uint64_t cnt_sent_in; // Counts message sent to child.
|
uint64_t cnt_sent_in; // Counts message sent to child.
|
||||||
uint64_t cnt_sent_out; // Counts message sent by child.
|
uint64_t cnt_sent_out; // Counts message sent by child.
|
||||||
|
|
||||||
|
bool finished; // Set to true by Finished message.
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,20 @@ bool Field::Write(SerializationFormat* fmt) const
|
||||||
return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
|
return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string Field::TypeName() const
|
||||||
|
{
|
||||||
|
string n = type_name(type);
|
||||||
|
|
||||||
|
if ( (type == TYPE_TABLE) || (type == TYPE_VECTOR) )
|
||||||
|
{
|
||||||
|
n += "[";
|
||||||
|
n += type_name(subtype);
|
||||||
|
n += "]";
|
||||||
|
}
|
||||||
|
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
Value::~Value()
|
Value::~Value()
|
||||||
{
|
{
|
||||||
if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
|
if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,12 @@ struct Field {
|
||||||
* @return False if an error occured.
|
* @return False if an error occured.
|
||||||
*/
|
*/
|
||||||
bool Write(SerializationFormat* fmt) const;
|
bool Write(SerializationFormat* fmt) const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a textual description of the field's type. This method is
|
||||||
|
* thread-safe.
|
||||||
|
*/
|
||||||
|
string TypeName() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -132,8 +138,8 @@ struct Value {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns true if the type can be represented by a Value. If
|
* Returns true if the type can be represented by a Value. If
|
||||||
* `atomic_only` is true, will not permit composite types.
|
* `atomic_only` is true, will not permit composite types. This
|
||||||
*/
|
* method is thread-safe. */
|
||||||
static bool IsCompatibleType(BroType* t, bool atomic_only=false);
|
static bool IsCompatibleType(BroType* t, bool atomic_only=false);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
|
||||||
|
|
@ -162,6 +162,7 @@ enum Writer %{
|
||||||
WRITER_DEFAULT,
|
WRITER_DEFAULT,
|
||||||
WRITER_NONE,
|
WRITER_NONE,
|
||||||
WRITER_ASCII,
|
WRITER_ASCII,
|
||||||
|
WRITER_DATASERIES,
|
||||||
%}
|
%}
|
||||||
|
|
||||||
enum ID %{
|
enum ID %{
|
||||||
|
|
|
||||||
|
|
@ -19,4 +19,5 @@ scripts/base/init-bare.bro
|
||||||
scripts/base/frameworks/logging/./postprocessors/./scp.bro
|
scripts/base/frameworks/logging/./postprocessors/./scp.bro
|
||||||
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
|
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
|
||||||
scripts/base/frameworks/logging/./writers/ascii.bro
|
scripts/base/frameworks/logging/./writers/ascii.bro
|
||||||
|
scripts/base/frameworks/logging/./writers/dataseries.bro
|
||||||
scripts/policy/misc/loaded-scripts.bro
|
scripts/policy/misc/loaded-scripts.bro
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ scripts/base/init-bare.bro
|
||||||
scripts/base/frameworks/logging/./postprocessors/./scp.bro
|
scripts/base/frameworks/logging/./postprocessors/./scp.bro
|
||||||
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
|
scripts/base/frameworks/logging/./postprocessors/./sftp.bro
|
||||||
scripts/base/frameworks/logging/./writers/ascii.bro
|
scripts/base/frameworks/logging/./writers/ascii.bro
|
||||||
|
scripts/base/frameworks/logging/./writers/dataseries.bro
|
||||||
scripts/base/init-default.bro
|
scripts/base/init-default.bro
|
||||||
scripts/base/utils/site.bro
|
scripts/base/utils/site.bro
|
||||||
scripts/base/utils/./patterns.bro
|
scripts/base/utils/./patterns.bro
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
<field type="variable32" name="status" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="country" pack_unique="yes"/>
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
<!-- status : string -->
|
||||||
|
<!-- country : string -->
|
||||||
|
|
@ -0,0 +1,290 @@
|
||||||
|
test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries
|
||||||
|
test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries
|
||||||
|
test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries
|
||||||
|
test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries
|
||||||
|
test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries
|
||||||
|
test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries
|
||||||
|
test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries
|
||||||
|
test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries
|
||||||
|
test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries
|
||||||
|
test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries
|
||||||
|
> test.2011-03-07-03-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299466805.000000 10.0.0.1 20 10.0.0.2 1024
|
||||||
|
1299470395.000000 10.0.0.2 20 10.0.0.3 0
|
||||||
|
> test.2011-03-07-04-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299470405.000000 10.0.0.1 20 10.0.0.2 1025
|
||||||
|
1299473995.000000 10.0.0.2 20 10.0.0.3 1
|
||||||
|
> test.2011-03-07-05-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299474005.000000 10.0.0.1 20 10.0.0.2 1026
|
||||||
|
1299477595.000000 10.0.0.2 20 10.0.0.3 2
|
||||||
|
> test.2011-03-07-06-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299477605.000000 10.0.0.1 20 10.0.0.2 1027
|
||||||
|
1299481195.000000 10.0.0.2 20 10.0.0.3 3
|
||||||
|
> test.2011-03-07-07-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299481205.000000 10.0.0.1 20 10.0.0.2 1028
|
||||||
|
1299484795.000000 10.0.0.2 20 10.0.0.3 4
|
||||||
|
> test.2011-03-07-08-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299484805.000000 10.0.0.1 20 10.0.0.2 1029
|
||||||
|
1299488395.000000 10.0.0.2 20 10.0.0.3 5
|
||||||
|
> test.2011-03-07-09-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299488405.000000 10.0.0.1 20 10.0.0.2 1030
|
||||||
|
1299491995.000000 10.0.0.2 20 10.0.0.3 6
|
||||||
|
> test.2011-03-07-10-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299492005.000000 10.0.0.1 20 10.0.0.2 1031
|
||||||
|
1299495595.000000 10.0.0.2 20 10.0.0.3 7
|
||||||
|
> test.2011-03-07-11-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299495605.000000 10.0.0.1 20 10.0.0.2 1032
|
||||||
|
1299499195.000000 10.0.0.2 20 10.0.0.3 8
|
||||||
|
> test.2011-03-07-12-00-05.ds
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
|
||||||
|
# Extent, type='test'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p
|
||||||
|
1299499205.000000 10.0.0.1 20 10.0.0.2 1033
|
||||||
|
1299502795.000000 10.0.0.2 20 10.0.0.3 9
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
<field type="variable32" name="status" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="country" pack_unique="yes"/>
|
||||||
|
</ExtentType>
|
||||||
|
<!-- t : time -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
<!-- status : string -->
|
||||||
|
<!-- country : string -->
|
||||||
|
|
||||||
|
# Extent, type='ssh'
|
||||||
|
t id.orig_h id.orig_p id.resp_h id.resp_p status country
|
||||||
|
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success unknown
|
||||||
|
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure US
|
||||||
|
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure UK
|
||||||
|
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success BR
|
||||||
|
1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure MX
|
||||||
|
|
@ -0,0 +1,87 @@
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="conn" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="int64" name="ts" pack_relative="ts" units="microseconds" epoch="unix"/>
|
||||||
|
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
<field type="variable32" name="proto" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="service" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="duration" pack_relative="duration" units="microseconds" epoch="unix"/>
|
||||||
|
<field type="int64" name="orig_bytes" />
|
||||||
|
<field type="int64" name="resp_bytes" />
|
||||||
|
<field type="variable32" name="conn_state" pack_unique="yes"/>
|
||||||
|
<field type="bool" name="local_orig" />
|
||||||
|
<field type="int64" name="missed_bytes" />
|
||||||
|
<field type="variable32" name="history" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="orig_pkts" />
|
||||||
|
<field type="int64" name="orig_ip_bytes" />
|
||||||
|
<field type="int64" name="resp_pkts" />
|
||||||
|
<field type="int64" name="resp_ip_bytes" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- ts : time -->
|
||||||
|
<!-- uid : string -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
<!-- proto : enum -->
|
||||||
|
<!-- service : string -->
|
||||||
|
<!-- duration : interval -->
|
||||||
|
<!-- orig_bytes : count -->
|
||||||
|
<!-- resp_bytes : count -->
|
||||||
|
<!-- conn_state : string -->
|
||||||
|
<!-- local_orig : bool -->
|
||||||
|
<!-- missed_bytes : count -->
|
||||||
|
<!-- history : string -->
|
||||||
|
<!-- orig_pkts : count -->
|
||||||
|
<!-- orig_ip_bytes : count -->
|
||||||
|
<!-- resp_pkts : count -->
|
||||||
|
<!-- resp_ip_bytes : count -->
|
||||||
|
|
||||||
|
# Extent, type='conn'
|
||||||
|
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||||
|
1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0
|
||||||
|
1300475167097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0
|
||||||
|
1300475167099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0
|
||||||
|
1300475168853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 435 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 420 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 391 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 317 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 343 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 375 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 339 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 334 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 420 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 384 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 316 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 422 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 402 0 131 SHR F 0 Cd 0 0 1 159
|
||||||
|
1300475168902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 374 0 198 SHR F 0 Cd 0 0 1 226
|
||||||
|
1300475169899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0
|
||||||
|
1300475170862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2613016 350 0 S0 F 0 D 7 546 0 0
|
||||||
|
1300475171675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 100096 66 0 S0 F 0 D 2 162 0 0
|
||||||
|
1300475171677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 100020 66 0 S0 F 0 D 2 122 0 0
|
||||||
|
1300475173116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 99801 66 0 S0 F 0 D 2 162 0 0
|
||||||
|
1300475173117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 99848 66 0 S0 F 0 D 2 122 0 0
|
||||||
|
1300475173153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0
|
||||||
|
1300475168859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 215893 1130 734 S1 F 1130 ShACad 4 216 4 950
|
||||||
|
1300475168652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 61328 0 350 OTH F 0 CdA 1 52 1 402
|
||||||
|
1300475168895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 227283 1178 734 S1 F 1178 ShACad 4 216 4 950
|
||||||
|
1300475168902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 120040 534 412 S1 F 534 ShACad 3 164 3 576
|
||||||
|
1300475168892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 229603 1148 734 S1 F 1148 ShACad 4 216 4 950
|
||||||
|
1300475168855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 218501 1171 733 S1 F 1171 ShACad 4 216 4 949
|
||||||
|
1300475168892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 220960 1137 733 S1 F 1137 ShACad 4 216 4 949
|
||||||
|
1300475169780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48
|
||||||
|
1300475168724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 119904 525 232 S1 F 525 ShACad 3 164 3 396
|
||||||
|
1300475168855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 219720 1125 734 S1 F 1125 ShACad 4 216 4 950
|
||||||
|
|
@ -0,0 +1,87 @@
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="conn" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
<field type="variable32" name="proto" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="service" pack_unique="yes"/>
|
||||||
|
<field type="double" name="duration" pack_relative="duration" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="int64" name="orig_bytes" />
|
||||||
|
<field type="int64" name="resp_bytes" />
|
||||||
|
<field type="variable32" name="conn_state" pack_unique="yes"/>
|
||||||
|
<field type="bool" name="local_orig" />
|
||||||
|
<field type="int64" name="missed_bytes" />
|
||||||
|
<field type="variable32" name="history" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="orig_pkts" />
|
||||||
|
<field type="int64" name="orig_ip_bytes" />
|
||||||
|
<field type="int64" name="resp_pkts" />
|
||||||
|
<field type="int64" name="resp_ip_bytes" />
|
||||||
|
</ExtentType>
|
||||||
|
<!-- ts : time -->
|
||||||
|
<!-- uid : string -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
<!-- proto : enum -->
|
||||||
|
<!-- service : string -->
|
||||||
|
<!-- duration : interval -->
|
||||||
|
<!-- orig_bytes : count -->
|
||||||
|
<!-- resp_bytes : count -->
|
||||||
|
<!-- conn_state : string -->
|
||||||
|
<!-- local_orig : bool -->
|
||||||
|
<!-- missed_bytes : count -->
|
||||||
|
<!-- history : string -->
|
||||||
|
<!-- orig_pkts : count -->
|
||||||
|
<!-- orig_ip_bytes : count -->
|
||||||
|
<!-- resp_pkts : count -->
|
||||||
|
<!-- resp_ip_bytes : count -->
|
||||||
|
|
||||||
|
# Extent, type='conn'
|
||||||
|
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||||
|
1300475167.096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
|
||||||
|
1300475167.097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
|
||||||
|
1300475167.099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
|
||||||
|
1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168.857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168.858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168.858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168.891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168.892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168.892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168.893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 0 89 SHR F 0 Cd 0 0 1 117
|
||||||
|
1300475168.894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 0 99 SHR F 0 Cd 0 0 1 127
|
||||||
|
1300475168.894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 0 183 SHR F 0 Cd 0 0 1 211
|
||||||
|
1300475168.901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 0 131 SHR F 0 Cd 0 0 1 159
|
||||||
|
1300475168.902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 0 198 SHR F 0 Cd 0 0 1 226
|
||||||
|
1300475169.899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0
|
||||||
|
1300475170.862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0
|
||||||
|
1300475171.675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0
|
||||||
|
1300475171.677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 F 0 D 2 122 0 0
|
||||||
|
1300475173.116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0
|
||||||
|
1300475173.117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0
|
||||||
|
1300475173.153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0
|
||||||
|
1300475168.859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0.215893 1130 734 S1 F 1130 ShACad 4 216 4 950
|
||||||
|
1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 0 350 OTH F 0 CdA 1 52 1 402
|
||||||
|
1300475168.895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0.227284 1178 734 S1 F 1178 ShACad 4 216 4 950
|
||||||
|
1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0.120041 534 412 S1 F 534 ShACad 3 164 3 576
|
||||||
|
1300475168.892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0.229603 1148 734 S1 F 1148 ShACad 4 216 4 950
|
||||||
|
1300475168.855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0.218501 1171 733 S1 F 1171 ShACad 4 216 4 949
|
||||||
|
1300475168.892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0.220961 1137 733 S1 F 1137 ShACad 4 216 4 949
|
||||||
|
1300475169.780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48
|
||||||
|
1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0.119905 525 232 S1 F 525 ShACad 3 164 3 396
|
||||||
|
1300475168.855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0.219720 1125 734 S1 F 1125 ShACad 4 216 4 950
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
# Extent Types ...
|
||||||
|
<ExtentType name="DataSeries: ExtentIndex">
|
||||||
|
<field type="int64" name="offset" />
|
||||||
|
<field type="variable32" name="extenttype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="DataSeries: XmlType">
|
||||||
|
<field type="variable32" name="xmltype" />
|
||||||
|
</ExtentType>
|
||||||
|
|
||||||
|
<ExtentType name="http" version="1.0" namespace="bro-ids.org">
|
||||||
|
<field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
|
||||||
|
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.orig_p" />
|
||||||
|
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="id.resp_p" />
|
||||||
|
<field type="int64" name="trans_depth" />
|
||||||
|
<field type="variable32" name="method" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="host" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="uri" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="referrer" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="user_agent" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="request_body_len" />
|
||||||
|
<field type="int64" name="response_body_len" />
|
||||||
|
<field type="int64" name="status_code" />
|
||||||
|
<field type="variable32" name="status_msg" pack_unique="yes"/>
|
||||||
|
<field type="int64" name="info_code" />
|
||||||
|
<field type="variable32" name="info_msg" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="filename" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="tags" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="username" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="password" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="proxied" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="mime_type" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="md5" pack_unique="yes"/>
|
||||||
|
<field type="variable32" name="extraction_file" pack_unique="yes"/>
|
||||||
|
</ExtentType>
|
||||||
|
<!-- ts : time -->
|
||||||
|
<!-- uid : string -->
|
||||||
|
<!-- id.orig_h : addr -->
|
||||||
|
<!-- id.orig_p : port -->
|
||||||
|
<!-- id.resp_h : addr -->
|
||||||
|
<!-- id.resp_p : port -->
|
||||||
|
<!-- trans_depth : count -->
|
||||||
|
<!-- method : string -->
|
||||||
|
<!-- host : string -->
|
||||||
|
<!-- uri : string -->
|
||||||
|
<!-- referrer : string -->
|
||||||
|
<!-- user_agent : string -->
|
||||||
|
<!-- request_body_len : count -->
|
||||||
|
<!-- response_body_len : count -->
|
||||||
|
<!-- status_code : count -->
|
||||||
|
<!-- status_msg : string -->
|
||||||
|
<!-- info_code : count -->
|
||||||
|
<!-- info_msg : string -->
|
||||||
|
<!-- filename : string -->
|
||||||
|
<!-- tags : table[enum] -->
|
||||||
|
<!-- username : string -->
|
||||||
|
<!-- password : string -->
|
||||||
|
<!-- proxied : table[string] -->
|
||||||
|
<!-- mime_type : string -->
|
||||||
|
<!-- md5 : string -->
|
||||||
|
<!-- extraction_file : file -->
|
||||||
|
|
||||||
|
# Extent, type='http'
|
||||||
|
ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
|
||||||
|
1300475168.843894 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475168.975800 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475168.976327 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475168.979160 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.012666 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.012730 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.014860 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.022665 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.036294 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.036798 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.039923 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.074793 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.074938 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
1300475169.075065 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0
|
1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii
|
||||||
1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0
|
1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii
|
||||||
1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0
|
1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii
|
||||||
1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0
|
1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii
|
||||||
1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0
|
1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii
|
||||||
1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0
|
1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii
|
||||||
1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0
|
1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii
|
||||||
1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0
|
1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii
|
||||||
1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
|
1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii
|
||||||
1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
|
1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,13 @@
|
||||||
test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0
|
test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii
|
||||||
test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0
|
test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii
|
||||||
test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0
|
test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii
|
||||||
test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0
|
test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii
|
||||||
test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0
|
test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii
|
||||||
test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0
|
test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii
|
||||||
test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0
|
test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii
|
||||||
test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0
|
test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii
|
||||||
test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
|
test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii
|
||||||
test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
|
test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
|
||||||
> test.2011-03-07-03-00-05.log
|
> test.2011-03-07-03-00-05.log
|
||||||
#separator \x09
|
#separator \x09
|
||||||
#set_separator ,
|
#set_separator ,
|
||||||
|
|
|
||||||
35
testing/btest/core/leaks/dataseries-rotate.bro
Normal file
35
testing/btest/core/leaks/dataseries-rotate.bro
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
|
||||||
|
#
|
||||||
|
# @TEST-GROUP: leaks
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/rotation.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
|
||||||
|
module Test;
|
||||||
|
|
||||||
|
export {
|
||||||
|
# Create a new ID for our log stream
|
||||||
|
redef enum Log::ID += { LOG };
|
||||||
|
|
||||||
|
# Define a record with all the columns the log file can have.
|
||||||
|
# (I'm using a subset of fields from ssh-ext for demonstration.)
|
||||||
|
type Log: record {
|
||||||
|
t: time;
|
||||||
|
id: conn_id; # Will be rolled out into individual columns.
|
||||||
|
} &log;
|
||||||
|
}
|
||||||
|
|
||||||
|
redef Log::default_rotation_interval = 1hr;
|
||||||
|
redef Log::default_rotation_postprocessor_cmd = "echo";
|
||||||
|
|
||||||
|
event bro_init()
|
||||||
|
{
|
||||||
|
Log::create_stream(Test::LOG, [$columns=Log]);
|
||||||
|
}
|
||||||
|
|
||||||
|
event new_connection(c: connection)
|
||||||
|
{
|
||||||
|
Log::write(Test::LOG, [$t=network_time(), $id=c$id]);
|
||||||
|
}
|
||||||
10
testing/btest/core/leaks/dataseries.bro
Normal file
10
testing/btest/core/leaks/dataseries.bro
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# Needs perftools support.
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
|
||||||
|
#
|
||||||
|
# @TEST-GROUP: leaks
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
|
||||||
|
# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
# @TEST-EXEC: test -e ssh.ds.xml
|
||||||
|
# @TEST-EXEC: btest-diff ssh.ds.xml
|
||||||
|
|
||||||
|
module SSH;
|
||||||
|
|
||||||
|
redef LogDataSeries::dump_schema = T;
|
||||||
|
|
||||||
|
# Haven't yet found a way to check for the effect of these.
|
||||||
|
redef LogDataSeries::compression = "bz2";
|
||||||
|
redef LogDataSeries::extent_size = 1000;
|
||||||
|
redef LogDataSeries::num_threads = 5;
|
||||||
|
|
||||||
|
# LogDataSeries::use_integer_for_time is tested separately.
|
||||||
|
|
||||||
|
export {
|
||||||
|
redef enum Log::ID += { LOG };
|
||||||
|
|
||||||
|
type Log: record {
|
||||||
|
t: time;
|
||||||
|
id: conn_id; # Will be rolled out into individual columns.
|
||||||
|
status: string &optional;
|
||||||
|
country: string &default="unknown";
|
||||||
|
} &log;
|
||||||
|
}
|
||||||
|
|
||||||
|
event bro_init()
|
||||||
|
{
|
||||||
|
Log::create_stream(SSH::LOG, [$columns=Log]);
|
||||||
|
|
||||||
|
local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
|
||||||
|
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out
|
||||||
|
# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt --skip-index $i; done >>out
|
||||||
|
# @TEST-EXEC: btest-diff out
|
||||||
|
|
||||||
|
module Test;
|
||||||
|
|
||||||
|
export {
|
||||||
|
# Create a new ID for our log stream
|
||||||
|
redef enum Log::ID += { LOG };
|
||||||
|
|
||||||
|
# Define a record with all the columns the log file can have.
|
||||||
|
# (I'm using a subset of fields from ssh-ext for demonstration.)
|
||||||
|
type Log: record {
|
||||||
|
t: time;
|
||||||
|
id: conn_id; # Will be rolled out into individual columns.
|
||||||
|
} &log;
|
||||||
|
}
|
||||||
|
|
||||||
|
redef Log::default_rotation_interval = 1hr;
|
||||||
|
redef Log::default_rotation_postprocessor_cmd = "echo";
|
||||||
|
|
||||||
|
event bro_init()
|
||||||
|
{
|
||||||
|
Log::create_stream(Test::LOG, [$columns=Log]);
|
||||||
|
}
|
||||||
|
|
||||||
|
event new_connection(c: connection)
|
||||||
|
{
|
||||||
|
Log::write(Test::LOG, [$t=network_time(), $id=c$id]);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
# @TEST-EXEC: ds2txt --skip-index ssh.ds >ssh.ds.txt
|
||||||
|
# @TEST-EXEC: btest-diff ssh.ds.txt
|
||||||
|
|
||||||
|
module SSH;
|
||||||
|
|
||||||
|
export {
|
||||||
|
redef enum Log::ID += { LOG };
|
||||||
|
|
||||||
|
type Log: record {
|
||||||
|
t: time;
|
||||||
|
id: conn_id; # Will be rolled out into individual columns.
|
||||||
|
status: string &optional;
|
||||||
|
country: string &default="unknown";
|
||||||
|
} &log;
|
||||||
|
}
|
||||||
|
|
||||||
|
event bro_init()
|
||||||
|
{
|
||||||
|
Log::create_stream(SSH::LOG, [$columns=Log]);
|
||||||
|
|
||||||
|
local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
|
||||||
|
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
|
||||||
|
Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt
|
||||||
|
# @TEST-EXEC: btest-diff conn.ds.txt
|
||||||
|
|
||||||
|
redef LogDataSeries::use_integer_for_time = T;
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
#
|
||||||
|
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||||
|
# @TEST-GROUP: dataseries
|
||||||
|
#
|
||||||
|
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
|
||||||
|
# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt
|
||||||
|
# @TEST-EXEC: ds2txt --skip-index http.ds >http.ds.txt
|
||||||
|
# @TEST-EXEC: btest-diff conn.ds.txt
|
||||||
|
# @TEST-EXEC: btest-diff http.ds.txt
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
#
|
#
|
||||||
# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT | egrep "test|test2" | sort >out
|
#@TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out
|
||||||
# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
|
# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
|
||||||
# @TEST-EXEC: btest-diff out
|
# @TEST-EXEC: btest-diff out
|
||||||
# @TEST-EXEC: btest-diff .stderr
|
# @TEST-EXEC: btest-diff .stderr
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
#
|
#
|
||||||
# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT 2>&1 | grep "test" >out
|
# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 | grep "test" >out
|
||||||
# @TEST-EXEC: for i in test.*.log; do printf '> %s\n' $i; cat $i; done >>out
|
# @TEST-EXEC: for i in `ls test.*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out
|
||||||
# @TEST-EXEC: btest-diff out
|
# @TEST-EXEC: btest-diff out
|
||||||
|
|
||||||
module Test;
|
module Test;
|
||||||
|
|
|
||||||
2
testing/external/subdir-btest.cfg
vendored
2
testing/external/subdir-btest.cfg
vendored
|
|
@ -10,7 +10,7 @@ BROPATH=`bash -c %(testbase)s/../../../build/bro-path-dev`:%(testbase)s/../scrip
|
||||||
BRO_SEED_FILE=%(testbase)s/../random.seed
|
BRO_SEED_FILE=%(testbase)s/../random.seed
|
||||||
TZ=UTC
|
TZ=UTC
|
||||||
LC_ALL=C
|
LC_ALL=C
|
||||||
PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(default_path)s
|
PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(testbase)s/../../scripts:%(default_path)s
|
||||||
TEST_DIFF_CANONIFIER=%(testbase)s/../../scripts/diff-canonifier-external
|
TEST_DIFF_CANONIFIER=%(testbase)s/../../scripts/diff-canonifier-external
|
||||||
TEST_DIFF_BRIEF=1
|
TEST_DIFF_BRIEF=1
|
||||||
TRACES=%(testbase)s/Traces
|
TRACES=%(testbase)s/Traces
|
||||||
|
|
|
||||||
6
testing/scripts/has-writer
Executable file
6
testing/scripts/has-writer
Executable file
|
|
@ -0,0 +1,6 @@
|
||||||
|
#! /usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Returns true if Bro has been compiled with support for writer type
|
||||||
|
# $1. The type name must match what "bro --help" prints.
|
||||||
|
|
||||||
|
bro --helper 2>&1 | grep -qi "Supported log formats:.*$1"
|
||||||
Loading…
Add table
Add a link
Reference in a new issue