Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-12 11:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Tweaking DataSeries support.

Browse source
This commit is contained in:
Robin Sommer 2012-04-19 10:41:24 -07:00
parent 18aa41c62b
commit 4b70adcb4b
10 changed files with 31 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

14
doc/logging-dataseries.rst
View file

@ -109,22 +109,8 @@ TODO
Warning, while packing field not_valid_after of record 11, error was > 10%: Warning, while packing field not_valid_after of record 11, error was > 10%:
(1346460000 / 1000000 = 1346.46, round() = 1346) (1346460000 / 1000000 = 1346.46, round() = 1346)
* The compiler warn about a depracated method and I'm not immediately
seeing how to avoid using that.
* For testing our script-level options: * For testing our script-level options:
- Can we get the extentsize from a ``.ds`` file? - Can we get the extentsize from a ``.ds`` file?
- Can we get the compressio level from a ``.ds`` file? - Can we get the compressio level from a ``.ds`` file?
* ds2txt can apparently not read a file that is currently being
written. That's not good for the spool directory::
# ds2txt http.ds
**** Assertion failure in file
/DataSeriesSink.cpp, line 301
**** Failed expression: tail[i] == 0xFF
**** Details: bad header for the tail of http.ds!
Can that be worked around?

10
src/logging/writers/DataSeries.cc
View file

@ -194,6 +194,8 @@ std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
if ( ! ds_use_integer_for_time ) if ( ! ds_use_integer_for_time )
s += " pack_scale=\"1000000\""; s += " pack_scale=\"1000000\"";
else
s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
return s; return s;
} }
@ -327,7 +329,13 @@ bool DataSeries::DoInit(string path, int num_fields, const threading::Field* con
else else
Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str())); Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str()));
log_type = log_types.registerType(schema); const ExtentType& type = log_types.registerTypeR(schema);
// Note: This is a bit dicey as it depends on the implementation of
// registerTypeR(), but its what the DataSeries guys recommended
// given that we function we originally used has been deprecated.
log_type = &type;
log_series.setType(*log_type); log_series.setType(*log_type);
return OpenLog(path); return OpenLog(path);

1
src/logging/writers/DataSeries.h
View file

@ -43,6 +43,7 @@ private:
static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn. static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn.
static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn. static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn.
static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers. static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers.
const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE.
struct SchemaValue struct SchemaValue
{ {

20
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out
View file

@ -1,13 +1,13 @@
test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries
test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries
test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries
test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries
test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries
test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries
test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries
test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries
test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries
test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries
> test.2011-03-07-03-00-05.ds > test.2011-03-07-03-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">

12
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="conn" version="1.0" namespace="bro-ids.org"> <ExtentType name="conn" version="1.0" namespace="bro-ids.org">
<field type="int64" name="ts" pack_relative="ts"/> <field type="int64" name="ts" pack_relative="ts" units="microseconds" epoch="unix"/>
<field type="variable32" name="uid" pack_unique="yes"/> <field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
@ -17,7 +17,7 @@
<field type="int64" name="id.resp_p" /> <field type="int64" name="id.resp_p" />
<field type="variable32" name="proto" pack_unique="yes"/> <field type="variable32" name="proto" pack_unique="yes"/>
<field type="variable32" name="service" pack_unique="yes"/> <field type="variable32" name="service" pack_unique="yes"/>
<field type="int64" name="duration" pack_relative="duration"/> <field type="int64" name="duration" pack_relative="duration" units="microseconds" epoch="unix"/>
<field type="int64" name="orig_bytes" /> <field type="int64" name="orig_bytes" />
<field type="int64" name="resp_bytes" /> <field type="int64" name="resp_bytes" />
<field type="variable32" name="conn_state" pack_unique="yes"/> <field type="variable32" name="conn_state" pack_unique="yes"/>
@ -51,8 +51,8 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
636 conn 672 conn
2912 DataSeries: ExtentIndex 2948 DataSeries: ExtentIndex
# Extent, type='conn' # Extent, type='conn'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0
@ -92,5 +92,5 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
636 conn 672 conn
2912 DataSeries: ExtentIndex 2948 DataSeries: ExtentIndex

1
testing/btest/scripts/base/frameworks/logging/dataseries/options.bro
View file

@ -1,5 +1,6 @@
# #
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt # @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-GROUP: dataseries
# #
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
# @TEST-EXEC: test -e ssh.ds.xml # @TEST-EXEC: test -e ssh.ds.xml

1
testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro
View file

@ -1,5 +1,6 @@
# #
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt # @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-GROUP: dataseries
# #
# @TEST-EXEC: bro -b -r %DIR/../rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out # @TEST-EXEC: bro -b -r %DIR/../rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out
# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt $i; done >>out # @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt $i; done >>out

1
testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro
View file

@ -1,5 +1,6 @@
# #
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt # @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-GROUP: dataseries
# #
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
# @TEST-EXEC: ds2txt ssh.ds | ${SCRIPTS}/diff-remove-timestamps-dataseries >ssh.ds.txt # @TEST-EXEC: ds2txt ssh.ds | ${SCRIPTS}/diff-remove-timestamps-dataseries >ssh.ds.txt

1
testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro
View file

@ -1,5 +1,6 @@
# #
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt # @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-GROUP: dataseries
# #
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
# @TEST-EXEC: ds2txt conn.ds >conn.ds.txt # @TEST-EXEC: ds2txt conn.ds >conn.ds.txt

1
testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro
View file

@ -1,5 +1,6 @@
# #
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt # @TEST-REQUIRES: has-writer DataSeries && which ds2txt
# @TEST-GROUP: dataseries
# #
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
# @TEST-EXEC: ds2txt conn.ds >conn.ds.txt # @TEST-EXEC: ds2txt conn.ds >conn.ds.txt