mirror of
https://github.com/zeek/zeek.git
synced 2025-10-11 19:18:19 +00:00
Tweaking DataSeries support.
This commit is contained in:
Robin Sommer
parent
18aa41c62b
commit
4b70adcb4b
10 changed files with 31 additions and 31 deletions
|
|
@ -109,22 +109,8 @@ TODO
|
|||
Warning, while packing field not_valid_after of record 11, error was > 10%:
|
||||
(1346460000 / 1000000 = 1346.46, round() = 1346)
|
||||
|
||||
* The compiler warn about a depracated method and I'm not immediately
|
||||
seeing how to avoid using that.
|
||||
|
||||
* For testing our script-level options:
|
||||
|
||||
- Can we get the extentsize from a ``.ds`` file?
|
||||
- Can we get the compressio level from a ``.ds`` file?
|
||||
|
||||
* ds2txt can apparently not read a file that is currently being
|
||||
written. That's not good for the spool directory::
|
||||
|
||||
# ds2txt http.ds
|
||||
**** Assertion failure in file
|
||||
/DataSeriesSink.cpp, line 301
|
||||
**** Failed expression: tail[i] == 0xFF
|
||||
**** Details: bad header for the tail of http.ds!
|
||||
|
||||
Can that be worked around?
|
||||
|
||||
|
|
|
|||
|
|
@ -194,6 +194,8 @@ std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
|
|||
|
||||
if ( ! ds_use_integer_for_time )
|
||||
s += " pack_scale=\"1000000\"";
|
||||
else
|
||||
s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
|
||||
|
||||
return s;
|
||||
}
|
||||
|
|
@ -327,7 +329,13 @@ bool DataSeries::DoInit(string path, int num_fields, const threading::Field* con
|
|||
else
|
||||
Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str()));
|
||||
|
||||
log_type = log_types.registerType(schema);
|
||||
const ExtentType& type = log_types.registerTypeR(schema);
|
||||
|
||||
// Note: This is a bit dicey as it depends on the implementation of
|
||||
// registerTypeR(), but its what the DataSeries guys recommended
|
||||
// given that we function we originally used has been deprecated.
|
||||
log_type = &type;
|
||||
|
||||
log_series.setType(*log_type);
|
||||
|
||||
return OpenLog(path);
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ private:
|
|||
static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn.
|
||||
static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn.
|
||||
static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers.
|
||||
const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE.
|
||||
|
||||
struct SchemaValue
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0
|
||||
test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0
|
||||
test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0
|
||||
test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0
|
||||
test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0
|
||||
test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0
|
||||
test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0
|
||||
test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0
|
||||
test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0
|
||||
test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1
|
||||
test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries
|
||||
test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries
|
||||
test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries
|
||||
test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries
|
||||
test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries
|
||||
test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries
|
||||
test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries
|
||||
test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries
|
||||
test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries
|
||||
test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries
|
||||
> test.2011-03-07-03-00-05.ds
|
||||
# Extent Types ...
|
||||
<ExtentType name="DataSeries: ExtentIndex">
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
</ExtentType>
|
||||
|
||||
<ExtentType name="conn" version="1.0" namespace="bro-ids.org">
|
||||
<field type="int64" name="ts" pack_relative="ts"/>
|
||||
<field type="int64" name="ts" pack_relative="ts" units="microseconds" epoch="unix"/>
|
||||
<field type="variable32" name="uid" pack_unique="yes"/>
|
||||
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
|
||||
<field type="int64" name="id.orig_p" />
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
<field type="int64" name="id.resp_p" />
|
||||
<field type="variable32" name="proto" pack_unique="yes"/>
|
||||
<field type="variable32" name="service" pack_unique="yes"/>
|
||||
<field type="int64" name="duration" pack_relative="duration"/>
|
||||
<field type="int64" name="duration" pack_relative="duration" units="microseconds" epoch="unix"/>
|
||||
<field type="int64" name="orig_bytes" />
|
||||
<field type="int64" name="resp_bytes" />
|
||||
<field type="variable32" name="conn_state" pack_unique="yes"/>
|
||||
|
|
@ -51,8 +51,8 @@
|
|||
|
||||
extent offset ExtentType
|
||||
40 DataSeries: XmlType
|
||||
636 conn
|
||||
2912 DataSeries: ExtentIndex
|
||||
672 conn
|
||||
2948 DataSeries: ExtentIndex
|
||||
# Extent, type='conn'
|
||||
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
|
||||
1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0
|
||||
|
|
@ -92,5 +92,5 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes
|
|||
# Extent, type='DataSeries: ExtentIndex'
|
||||
offset extenttype
|
||||
40 DataSeries: XmlType
|
||||
636 conn
|
||||
2912 DataSeries: ExtentIndex
|
||||
672 conn
|
||||
2948 DataSeries: ExtentIndex
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: test -e ssh.ds.xml
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -b -r %DIR/../rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out
|
||||
# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt $i; done >>out
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: ds2txt ssh.ds | ${SCRIPTS}/diff-remove-timestamps-dataseries >ssh.ds.txt
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: ds2txt conn.ds >conn.ds.txt
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#
|
||||
# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
|
||||
# @TEST-GROUP: dataseries
|
||||
#
|
||||
# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
|
||||
# @TEST-EXEC: ds2txt conn.ds >conn.ds.txt
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue