Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

DataSeries updates and fixes.

Browse source
This commit is contained in:
Robin Sommer 2012-05-04 21:50:20 -07:00
parent c91563fe75
commit a0575158ef
10 changed files with 169 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

16
doc/logging-dataseries.rst
View file

@ -102,20 +102,4 @@ TODO.
TODO TODO
==== ====
* I'm seeing lots of warning on stderr::
Warning, while packing field ts of record 1, error was > 10%:
(1334620000 / 1000000 = 1334.62, round() = 1335)
Warning, while packing field not_valid_after of record 11, error was > 10%:
(1346460000 / 1000000 = 1346.46, round() = 1346)
See Eric's mail.
* For testing our script-level options:
- Can we get the extentsize from a ``.ds`` file?
- Can we get the compressio level from a ``.ds`` file?
See Eric's mail.
* Do we have a leak? * Do we have a leak?

7
src/logging/WriterBackend.cc
View file

@ -267,4 +267,9 @@ string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
return s; return s;
} }
string WriterBackend::Render(double d) const
{
char buf[256];
modp_dtoa(d, buf, 6);
return buf;
}

8
src/logging/WriterBackend.h
View file

@ -165,6 +165,14 @@ public:
*/ */
string Render(const threading::Value::subnet_t& subnet) const; string Render(const threading::Value::subnet_t& subnet) const;
/** Helper method to render a double in Bro's standard precision.
*
* @param d The double.
*
* @return An ASCII representation of the double.
*/
string Render(double d) const;
protected: protected:
friend class FinishMessage; friend class FinishMessage;

7
src/logging/writers/Ascii.cc
View file

@ -176,14 +176,9 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
desc->Add(Render(val->val.addr_val)); desc->Add(Render(val->val.addr_val));
break; break;
case TYPE_DOUBLE:
case TYPE_TIME: case TYPE_TIME:
case TYPE_INTERVAL: case TYPE_INTERVAL:
char buf[256];
modp_dtoa(val->val.double_val, buf, 6);
desc->Add(buf);
break;
case TYPE_DOUBLE:
desc->Add(val->val.double_val); desc->Add(val->val.double_val);
break; break;

34
src/logging/writers/DataSeries.cc
View file

@ -21,29 +21,31 @@ std::string DataSeries::LogValueToString(threading::Value *val)
if( ! val->present ) if( ! val->present )
return ""; return "";
std::ostringstream ostr;
switch(val->type) { switch(val->type) {
case TYPE_BOOL: case TYPE_BOOL:
return (val->val.int_val ? "true" : "false"); return (val->val.int_val ? "true" : "false");
case TYPE_INT: case TYPE_INT:
{
std::ostringstream ostr;
ostr << val->val.int_val; ostr << val->val.int_val;
return ostr.str(); return ostr.str();
}
case TYPE_COUNT: case TYPE_COUNT:
case TYPE_COUNTER: case TYPE_COUNTER:
case TYPE_PORT: case TYPE_PORT:
{
std::ostringstream ostr;
ostr << val->val.uint_val; ostr << val->val.uint_val;
return ostr.str(); return ostr.str();
}
case TYPE_SUBNET: case TYPE_SUBNET:
ostr << Render(val->val.subnet_val); return Render(val->val.subnet_val);
return ostr.str();
case TYPE_ADDR: case TYPE_ADDR:
ostr << Render(val->val.addr_val); return Render(val->val.addr_val);
return ostr.str();
// Note: These two cases are relatively special. We need to convert // Note: These two cases are relatively special. We need to convert
// these values into their integer equivalents to maximize precision. // these values into their integer equivalents to maximize precision.
@ -57,15 +59,16 @@ std::string DataSeries::LogValueToString(threading::Value *val)
case TYPE_TIME: case TYPE_TIME:
case TYPE_INTERVAL: case TYPE_INTERVAL:
if ( ds_use_integer_for_time ) if ( ds_use_integer_for_time )
{
std::ostringstream ostr;
ostr << (unsigned long)(DataSeries::TIME_SCALE * val->val.double_val); ostr << (unsigned long)(DataSeries::TIME_SCALE * val->val.double_val);
else
ostr << val->val.double_val;
return ostr.str(); return ostr.str();
}
else
return Render(val->val.double_val);
case TYPE_DOUBLE: case TYPE_DOUBLE:
ostr << val->val.double_val; return Render(val->val.double_val);
return ostr.str();
case TYPE_ENUM: case TYPE_ENUM:
case TYPE_STRING: case TYPE_STRING:
@ -190,10 +193,11 @@ std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
case TYPE_TIME: case TYPE_TIME:
case TYPE_INTERVAL: case TYPE_INTERVAL:
{ {
std::string s = "pack_relative=\"" + std::string(field->name) + "\""; std::string s;
s += "pack_relative=\"" + std::string(field->name) + "\"";
if ( ! ds_use_integer_for_time ) if ( ! ds_use_integer_for_time )
s += " pack_scale=\"1000000\""; s += " pack_scale=\"1000\" pack_scale_warn=\"no\"";
else else
s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\""; s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
@ -250,7 +254,7 @@ bool DataSeries::OpenLog(string path)
ds_extent_size = ROW_MAX; ds_extent_size = ROW_MAX;
} }
log_output = new OutputModule(*log_file, log_series, *log_type, ds_extent_size); log_output = new OutputModule(*log_file, log_series, log_type, ds_extent_size);
return true; return true;
} }
@ -330,7 +334,7 @@ bool DataSeries::DoInit(string path, int num_fields, const threading::Field* con
Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str())); Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str()));
log_type = log_types.registerTypePtr(schema); log_type = log_types.registerTypePtr(schema);
log_series.setType(*log_type); log_series.setType(log_type);
return OpenLog(path); return OpenLog(path);
} }

2
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml
View file

@ -1,5 +1,5 @@
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org"> <ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>

140
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out
View file

@ -20,7 +20,7 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -34,17 +34,17 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1024 1.299467e+09 10.0.0.1 20 10.0.0.2 1024
1.299e+09 10.0.0.2 20 10.0.0.3 0 1.299471e+09 10.0.0.2 20 10.0.0.3 0
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-04-00-05.ds > test.2011-03-07-04-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -57,7 +57,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -71,17 +71,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1025 1.29947e+09 10.0.0.1 20 10.0.0.2 1025
1.299e+09 10.0.0.2 20 10.0.0.3 1 1.299474e+09 10.0.0.2 20 10.0.0.3 1
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-05-00-05.ds > test.2011-03-07-05-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -94,7 +94,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -108,17 +108,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1026 1.299474e+09 10.0.0.1 20 10.0.0.2 1026
1.299e+09 10.0.0.2 20 10.0.0.3 2 1.299478e+09 10.0.0.2 20 10.0.0.3 2
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-06-00-05.ds > test.2011-03-07-06-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -131,7 +131,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -145,17 +145,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1027 1.299478e+09 10.0.0.1 20 10.0.0.2 1027
1.299e+09 10.0.0.2 20 10.0.0.3 3 1.299482e+09 10.0.0.2 20 10.0.0.3 3
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-07-00-05.ds > test.2011-03-07-07-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -168,7 +168,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -182,17 +182,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1028 1.299481e+09 10.0.0.1 20 10.0.0.2 1028
1.299e+09 10.0.0.2 20 10.0.0.3 4 1.299485e+09 10.0.0.2 20 10.0.0.3 4
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-08-00-05.ds > test.2011-03-07-08-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -205,7 +205,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -219,17 +219,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1029 1.299485e+09 10.0.0.1 20 10.0.0.2 1029
1.299e+09 10.0.0.2 20 10.0.0.3 5 1.299489e+09 10.0.0.2 20 10.0.0.3 5
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-09-00-05.ds > test.2011-03-07-09-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -242,7 +242,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -256,17 +256,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1030 1.299488e+09 10.0.0.1 20 10.0.0.2 1030
1.299e+09 10.0.0.2 20 10.0.0.3 6 1.299492e+09 10.0.0.2 20 10.0.0.3 6
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-10-00-05.ds > test.2011-03-07-10-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -279,7 +279,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -293,17 +293,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299e+09 10.0.0.1 20 10.0.0.2 1031 1.299492e+09 10.0.0.1 20 10.0.0.2 1031
1.299e+09 10.0.0.2 20 10.0.0.3 7 1.299496e+09 10.0.0.2 20 10.0.0.3 7
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-11-00-05.ds > test.2011-03-07-11-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -316,7 +316,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -330,17 +330,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.3e+09 10.0.0.1 20 10.0.0.2 1032 1.299496e+09 10.0.0.1 20 10.0.0.2 1032
1.3e+09 10.0.0.2 20 10.0.0.3 8 1.2995e+09 10.0.0.2 20 10.0.0.3 8
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
> test.2011-03-07-12-00-05.ds > test.2011-03-07-12-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -353,7 +353,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -367,14 +367,14 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.3e+09 10.0.0.1 20 10.0.0.2 1033 1.299499e+09 10.0.0.1 20 10.0.0.2 1033
1.3e+09 10.0.0.2 20 10.0.0.3 9 1.299503e+09 10.0.0.2 20 10.0.0.3 9
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
360 test 372 test
468 DataSeries: ExtentIndex 484 DataSeries: ExtentIndex

10
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org"> <ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000000"/> <field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -27,8 +27,8 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
400 ssh 416 ssh
604 DataSeries: ExtentIndex 624 DataSeries: ExtentIndex
# Extent, type='ssh' # Extent, type='ssh'
t id.orig_h id.orig_p id.resp_h id.resp_p status country t id.orig_h id.orig_p id.resp_h id.resp_p status country
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 success unknown X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 success unknown
@ -39,5 +39,5 @@ X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure MX
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
400 ssh 416 ssh
604 DataSeries: ExtentIndex 624 DataSeries: ExtentIndex

80
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="conn" version="1.0" namespace="bro-ids.org"> <ExtentType name="conn" version="1.0" namespace="bro-ids.org">
<field type="double" name="ts" pack_relative="ts" pack_scale="1000000"/> <field type="double" name="ts" pack_relative="ts" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="uid" pack_unique="yes"/> <field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
@ -17,7 +17,7 @@
<field type="int64" name="id.resp_p" /> <field type="int64" name="id.resp_p" />
<field type="variable32" name="proto" pack_unique="yes"/> <field type="variable32" name="proto" pack_unique="yes"/>
<field type="variable32" name="service" pack_unique="yes"/> <field type="variable32" name="service" pack_unique="yes"/>
<field type="double" name="duration" pack_relative="duration" pack_scale="1000000"/> <field type="double" name="duration" pack_relative="duration" pack_scale="1000" pack_scale_warn="no"/>
<field type="int64" name="orig_bytes" /> <field type="int64" name="orig_bytes" />
<field type="int64" name="resp_bytes" /> <field type="int64" name="resp_bytes" />
<field type="variable32" name="conn_state" pack_unique="yes"/> <field type="variable32" name="conn_state" pack_unique="yes"/>
@ -51,46 +51,46 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
660 conn 680 conn
2564 DataSeries: ExtentIndex 2592 DataSeries: ExtentIndex
# Extent, type='conn' # Extent, type='conn'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
1.3e+09 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 1.300475e+09 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0
1.3e+09 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0 1.300475e+09 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0
1.3e+09 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0 1.300475e+09 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0
1.3e+09 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1.300475e+09 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.3e+09 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1.300475e+09 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.3e+09 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1.300475e+09 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.3e+09 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1.300475e+09 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.3e+09 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1.300475e+09 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.3e+09 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1.300475e+09 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.3e+09 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1.300475e+09 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.3e+09 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1.300475e+09 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.3e+09 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1.300475e+09 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.3e+09 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1.300475e+09 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.3e+09 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1.300475e+09 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.3e+09 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1.300475e+09 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.3e+09 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0 0 131 SHR F 0 Cd 0 0 1 159 1.300475e+09 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0 0 131 SHR F 0 Cd 0 0 1 159
1.3e+09 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0 0 198 SHR F 0 Cd 0 0 1 226 1.300475e+09 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0 0 198 SHR F 0 Cd 0 0 1 226
1.3e+09 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0 1.300475e+09 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0
1.3e+09 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 0 350 0 S0 F 0 D 7 546 0 0 1.300475e+09 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 0 350 0 S0 F 0 D 7 546 0 0
1.3e+09 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0 1.300475e+09 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0
1.3e+09 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0 1.300475e+09 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0
1.3e+09 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0 1.300475e+09 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0
1.3e+09 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0 1.300475e+09 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0
1.3e+09 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0 1.300475e+09 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0
1.3e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0 1130 734 S1 F 1130 ShACad 4 216 4 950 1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0 1130 734 S1 F 1130 ShACad 4 216 4 950
1.3e+09 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0 0 350 OTH F 0 CdA 1 52 1 402 1.300475e+09 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0 0 350 OTH F 0 CdA 1 52 1 402
1.3e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0 1178 734 S1 F 1178 ShACad 4 216 4 950 1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0 1178 734 S1 F 1178 ShACad 4 216 4 950
1.3e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0 534 412 S1 F 534 ShACad 3 164 3 576 1.300475e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0 534 412 S1 F 534 ShACad 3 164 3 576
1.3e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0 1148 734 S1 F 1148 ShACad 4 216 4 950 1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0 1148 734 S1 F 1148 ShACad 4 216 4 950
1.3e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0 1171 733 S1 F 1171 ShACad 4 216 4 949 1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0 1171 733 S1 F 1171 ShACad 4 216 4 949
1.3e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0 1137 733 S1 F 1137 ShACad 4 216 4 949 1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0 1137 733 S1 F 1137 ShACad 4 216 4 949
1.3e+09 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48 1.300475e+09 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48
1.3e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0 525 232 S1 F 525 ShACad 3 164 3 396 1.300475e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0 525 232 S1 F 525 ShACad 3 164 3 396
1.3e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0 1125 734 S1 F 1125 ShACad 4 216 4 950 1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0 1125 734 S1 F 1125 ShACad 4 216 4 950
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
660 conn 680 conn
2564 DataSeries: ExtentIndex 2592 DataSeries: ExtentIndex

38
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="http" version="1.0" namespace="bro-ids.org"> <ExtentType name="http" version="1.0" namespace="bro-ids.org">
<field type="double" name="ts" pack_relative="ts" pack_scale="1000000"/> <field type="double" name="ts" pack_relative="ts" pack_scale="1000" pack_scale_warn="no"/>
<field type="variable32" name="uid" pack_unique="yes"/> <field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
@ -65,26 +65,26 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
768 http 784 http
1156 DataSeries: ExtentIndex 1172 DataSeries: ExtentIndex
# Extent, type='http' # Extent, type='http'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
1.3e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0 1.300475e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0
1.3e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0 1.300475e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0
1.3e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1.3e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
768 http 784 http
1156 DataSeries: ExtentIndex 1172 DataSeries: ExtentIndex