Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Fixing pack_scale and time-as-int.

Browse source 
Also removing now unneccessary canonifier script, and updating test
baselines.
This commit is contained in:
Robin Sommer 2012-05-14 21:58:58 -07:00
parent 00b592f933
commit fabe891d4f
8 changed files with 144 additions and 151 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/logging/writers/DataSeries.cc
View file

@ -194,13 +194,12 @@ std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
case TYPE_INTERVAL: case TYPE_INTERVAL:
{ {
std::string s; std::string s;
s += "pack_relative=\"" + std::string(field->name) + "\" "; s += "pack_relative=\"" + std::string(field->name) + "\"";
s += "print_format=\"%.6f\" ";
if ( ! ds_use_integer_for_time ) if ( ! ds_use_integer_for_time )
s += "pack_scale=\"1000\" pack_scale_warn=\"no\""; s += " pack_scale=\"1e-6\" print_format=\"%.6f\" pack_scale_warn=\"no\"";
else else
s += string("units=\"") + TIME_UNIT() + "\" epoch=\"unix\""; s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
return s; return s;
} }

2
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml
View file

@ -1,5 +1,5 @@
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org"> <ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>

140
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out
View file

@ -20,7 +20,7 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -34,17 +34,17 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 508 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299467e+09 10.0.0.1 20 10.0.0.2 1024 1299466805.000000 10.0.0.1 20 10.0.0.2 1024
1.299471e+09 10.0.0.2 20 10.0.0.3 0 1299470395.000000 10.0.0.2 20 10.0.0.3 0
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 508 DataSeries: ExtentIndex
> test.2011-03-07-04-00-05.ds > test.2011-03-07-04-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -57,7 +57,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -71,17 +71,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.29947e+09 10.0.0.1 20 10.0.0.2 1025 1299470405.000000 10.0.0.1 20 10.0.0.2 1025
1.299474e+09 10.0.0.2 20 10.0.0.3 1 1299473995.000000 10.0.0.2 20 10.0.0.3 1
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-05-00-05.ds > test.2011-03-07-05-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -94,7 +94,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -108,17 +108,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299474e+09 10.0.0.1 20 10.0.0.2 1026 1299474005.000000 10.0.0.1 20 10.0.0.2 1026
1.299478e+09 10.0.0.2 20 10.0.0.3 2 1299477595.000000 10.0.0.2 20 10.0.0.3 2
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-06-00-05.ds > test.2011-03-07-06-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -131,7 +131,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -145,17 +145,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299478e+09 10.0.0.1 20 10.0.0.2 1027 1299477605.000000 10.0.0.1 20 10.0.0.2 1027
1.299482e+09 10.0.0.2 20 10.0.0.3 3 1299481195.000000 10.0.0.2 20 10.0.0.3 3
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-07-00-05.ds > test.2011-03-07-07-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -168,7 +168,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -182,17 +182,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 512 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299481e+09 10.0.0.1 20 10.0.0.2 1028 1299481205.000000 10.0.0.1 20 10.0.0.2 1028
1.299485e+09 10.0.0.2 20 10.0.0.3 4 1299484795.000000 10.0.0.2 20 10.0.0.3 4
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 512 DataSeries: ExtentIndex
> test.2011-03-07-08-00-05.ds > test.2011-03-07-08-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -205,7 +205,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -219,17 +219,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299485e+09 10.0.0.1 20 10.0.0.2 1029 1299484805.000000 10.0.0.1 20 10.0.0.2 1029
1.299489e+09 10.0.0.2 20 10.0.0.3 5 1299488395.000000 10.0.0.2 20 10.0.0.3 5
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-09-00-05.ds > test.2011-03-07-09-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -242,7 +242,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -256,17 +256,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299488e+09 10.0.0.1 20 10.0.0.2 1030 1299488405.000000 10.0.0.1 20 10.0.0.2 1030
1.299492e+09 10.0.0.2 20 10.0.0.3 6 1299491995.000000 10.0.0.2 20 10.0.0.3 6
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-10-00-05.ds > test.2011-03-07-10-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -279,7 +279,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -293,17 +293,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299492e+09 10.0.0.1 20 10.0.0.2 1031 1299492005.000000 10.0.0.1 20 10.0.0.2 1031
1.299496e+09 10.0.0.2 20 10.0.0.3 7 1299495595.000000 10.0.0.2 20 10.0.0.3 7
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-11-00-05.ds > test.2011-03-07-11-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -316,7 +316,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -330,17 +330,17 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299496e+09 10.0.0.1 20 10.0.0.2 1032 1299495605.000000 10.0.0.1 20 10.0.0.2 1032
1.2995e+09 10.0.0.2 20 10.0.0.3 8 1299499195.000000 10.0.0.2 20 10.0.0.3 8
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
> test.2011-03-07-12-00-05.ds > test.2011-03-07-12-00-05.ds
# Extent Types ... # Extent Types ...
<ExtentType name="DataSeries: ExtentIndex"> <ExtentType name="DataSeries: ExtentIndex">
@ -353,7 +353,7 @@ offset extenttype
</ExtentType> </ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org"> <ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -367,14 +367,14 @@ offset extenttype
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex
# Extent, type='test' # Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p t id.orig_h id.orig_p id.resp_h id.resp_p
1.299499e+09 10.0.0.1 20 10.0.0.2 1033 1299499205.000000 10.0.0.1 20 10.0.0.2 1033
1.299503e+09 10.0.0.2 20 10.0.0.3 9 1299502795.000000 10.0.0.2 20 10.0.0.3 9
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
372 test 392 test
484 DataSeries: ExtentIndex 516 DataSeries: ExtentIndex

20
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org"> <ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/> <field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -27,17 +27,17 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
416 ssh 436 ssh
624 DataSeries: ExtentIndex 644 DataSeries: ExtentIndex
# Extent, type='ssh' # Extent, type='ssh'
t id.orig_h id.orig_p id.resp_h id.resp_p status country t id.orig_h id.orig_p id.resp_h id.resp_p status country
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 success unknown 1337058239.030366 1.2.3.4 1234 2.3.4.5 80 success unknown
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure US 1337058239.030366 1.2.3.4 1234 2.3.4.5 80 failure US
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure UK 1337058239.030366 1.2.3.4 1234 2.3.4.5 80 failure UK
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 success BR 1337058239.030366 1.2.3.4 1234 2.3.4.5 80 success BR
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure MX 1337058239.030366 1.2.3.4 1234 2.3.4.5 80 failure MX
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
416 ssh 436 ssh
624 DataSeries: ExtentIndex 644 DataSeries: ExtentIndex

80
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="conn" version="1.0" namespace="bro-ids.org"> <ExtentType name="conn" version="1.0" namespace="bro-ids.org">
<field type="double" name="ts" pack_relative="ts" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="uid" pack_unique="yes"/> <field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
@ -17,7 +17,7 @@
<field type="int64" name="id.resp_p" /> <field type="int64" name="id.resp_p" />
<field type="variable32" name="proto" pack_unique="yes"/> <field type="variable32" name="proto" pack_unique="yes"/>
<field type="variable32" name="service" pack_unique="yes"/> <field type="variable32" name="service" pack_unique="yes"/>
<field type="double" name="duration" pack_relative="duration" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="duration" pack_relative="duration" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="int64" name="orig_bytes" /> <field type="int64" name="orig_bytes" />
<field type="int64" name="resp_bytes" /> <field type="int64" name="resp_bytes" />
<field type="variable32" name="conn_state" pack_unique="yes"/> <field type="variable32" name="conn_state" pack_unique="yes"/>
@ -51,46 +51,46 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
680 conn 700 conn
2592 DataSeries: ExtentIndex 2860 DataSeries: ExtentIndex
# Extent, type='conn' # Extent, type='conn'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
1.300475e+09 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 1300475167.096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
1.300475e+09 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0 1300475167.097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
1.300475e+09 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0 1300475167.099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
1.300475e+09 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1300475168.857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1300475168.858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1300475168.858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1300475168.891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1300475168.892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1300475168.892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117 1300475168.893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127 1300475168.894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211 1300475168.894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0 0 131 SHR F 0 Cd 0 0 1 159 1300475168.901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 0 131 SHR F 0 Cd 0 0 1 159
1.300475e+09 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0 0 198 SHR F 0 Cd 0 0 1 226 1300475168.902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 0 198 SHR F 0 Cd 0 0 1 226
1.300475e+09 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0 1300475169.899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0
1.300475e+09 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 0 350 0 S0 F 0 D 7 546 0 0 1300475170.862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0
1.300475e+09 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0 1300475171.675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0
1.300475e+09 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0 1300475171.677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 F 0 D 2 122 0 0
1.300475e+09 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0 1300475173.116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0
1.300475e+09 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0 1300475173.117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0
1.300475e+09 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0 1300475173.153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0
1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0 1130 734 S1 F 1130 ShACad 4 216 4 950 1300475168.859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0.215893 1130 734 S1 F 1130 ShACad 4 216 4 950
1.300475e+09 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0 0 350 OTH F 0 CdA 1 52 1 402 1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 0 350 OTH F 0 CdA 1 52 1 402
1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0 1178 734 S1 F 1178 ShACad 4 216 4 950 1300475168.895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0.227284 1178 734 S1 F 1178 ShACad 4 216 4 950
1.300475e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0 534 412 S1 F 534 ShACad 3 164 3 576 1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0.120041 534 412 S1 F 534 ShACad 3 164 3 576
1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0 1148 734 S1 F 1148 ShACad 4 216 4 950 1300475168.892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0.229603 1148 734 S1 F 1148 ShACad 4 216 4 950
1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0 1171 733 S1 F 1171 ShACad 4 216 4 949 1300475168.855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0.218501 1171 733 S1 F 1171 ShACad 4 216 4 949
1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0 1137 733 S1 F 1137 ShACad 4 216 4 949 1300475168.892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0.220961 1137 733 S1 F 1137 ShACad 4 216 4 949
1.300475e+09 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48 1300475169.780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48
1.300475e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0 525 232 S1 F 525 ShACad 3 164 3 396 1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0.119905 525 232 S1 F 525 ShACad 3 164 3 396
1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0 1125 734 S1 F 1125 ShACad 4 216 4 950 1300475168.855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0.219720 1125 734 S1 F 1125 ShACad 4 216 4 950
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
680 conn 700 conn
2592 DataSeries: ExtentIndex 2860 DataSeries: ExtentIndex

38
testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt
View file

@ -9,7 +9,7 @@
</ExtentType> </ExtentType>
<ExtentType name="http" version="1.0" namespace="bro-ids.org"> <ExtentType name="http" version="1.0" namespace="bro-ids.org">
<field type="double" name="ts" pack_relative="ts" pack_scale="1000" pack_scale_warn="no"/> <field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="uid" pack_unique="yes"/> <field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/> <field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" /> <field type="int64" name="id.orig_p" />
@ -65,26 +65,26 @@
extent offset ExtentType extent offset ExtentType
40 DataSeries: XmlType 40 DataSeries: XmlType
784 http 804 http
1172 DataSeries: ExtentIndex 1252 DataSeries: ExtentIndex
# Extent, type='http' # Extent, type='http'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
1.300475e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0 1300475168.843894 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0
1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475168.975800 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475168.976327 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475168.979160 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.012666 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.012730 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.014860 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0 1300475169.022665 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0
1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.036294 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.036798 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.039923 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.074793 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.074938 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 1300475169.075065 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
# Extent, type='DataSeries: ExtentIndex' # Extent, type='DataSeries: ExtentIndex'
offset extenttype offset extenttype
40 DataSeries: XmlType 40 DataSeries: XmlType
784 http 804 http
1172 DataSeries: ExtentIndex 1252 DataSeries: ExtentIndex

2
testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro
View file

@ -3,7 +3,7 @@
# @TEST-GROUP: dataseries # @TEST-GROUP: dataseries
# #
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
# @TEST-EXEC: ds2txt ssh.ds | ${SCRIPTS}/diff-remove-timestamps-dataseries >ssh.ds.txt # @TEST-EXEC: ds2txt ssh.ds >ssh.ds.txt
# @TEST-EXEC: btest-diff ssh.ds.txt # @TEST-EXEC: btest-diff ssh.ds.txt
module SSH; module SSH;

6
testing/scripts/diff-remove-timestamps-dataseries
View file

@ -1,6 +0,0 @@
#! /usr/bin/env bash
#
# Replace anything which looks like DataSeries timestamps (which is a double) with XXXs.
sed 's/1\.[0-9]*e+09/X.XXXe+09/g'