Fixing pack_scale and time-as-int.

Also removing now unneccessary canonifier script, and updating test
baselines.
This commit is contained in:
Robin Sommer 2012-05-14 21:58:58 -07:00
parent 00b592f933
commit fabe891d4f
8 changed files with 144 additions and 151 deletions

View file

@ -194,13 +194,12 @@ std::string DataSeries::GetDSOptionsForType(const threading::Field *field)
case TYPE_INTERVAL:
{
std::string s;
s += "pack_relative=\"" + std::string(field->name) + "\" ";
s += "print_format=\"%.6f\" ";
s += "pack_relative=\"" + std::string(field->name) + "\"";
if ( ! ds_use_integer_for_time )
s += "pack_scale=\"1000\" pack_scale_warn=\"no\"";
s += " pack_scale=\"1e-6\" print_format=\"%.6f\" pack_scale_warn=\"no\"";
else
s += string("units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\"";
return s;
}

View file

@ -1,5 +1,5 @@
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>

View file

@ -20,7 +20,7 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -34,17 +34,17 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
508 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299467e+09 10.0.0.1 20 10.0.0.2 1024
1.299471e+09 10.0.0.2 20 10.0.0.3 0
1299466805.000000 10.0.0.1 20 10.0.0.2 1024
1299470395.000000 10.0.0.2 20 10.0.0.3 0
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
508 DataSeries: ExtentIndex
> test.2011-03-07-04-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -57,7 +57,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -71,17 +71,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.29947e+09 10.0.0.1 20 10.0.0.2 1025
1.299474e+09 10.0.0.2 20 10.0.0.3 1
1299470405.000000 10.0.0.1 20 10.0.0.2 1025
1299473995.000000 10.0.0.2 20 10.0.0.3 1
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-05-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -94,7 +94,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -108,17 +108,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299474e+09 10.0.0.1 20 10.0.0.2 1026
1.299478e+09 10.0.0.2 20 10.0.0.3 2
1299474005.000000 10.0.0.1 20 10.0.0.2 1026
1299477595.000000 10.0.0.2 20 10.0.0.3 2
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-06-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -131,7 +131,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -145,17 +145,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299478e+09 10.0.0.1 20 10.0.0.2 1027
1.299482e+09 10.0.0.2 20 10.0.0.3 3
1299477605.000000 10.0.0.1 20 10.0.0.2 1027
1299481195.000000 10.0.0.2 20 10.0.0.3 3
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-07-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -168,7 +168,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -182,17 +182,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
512 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299481e+09 10.0.0.1 20 10.0.0.2 1028
1.299485e+09 10.0.0.2 20 10.0.0.3 4
1299481205.000000 10.0.0.1 20 10.0.0.2 1028
1299484795.000000 10.0.0.2 20 10.0.0.3 4
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
512 DataSeries: ExtentIndex
> test.2011-03-07-08-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -205,7 +205,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -219,17 +219,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299485e+09 10.0.0.1 20 10.0.0.2 1029
1.299489e+09 10.0.0.2 20 10.0.0.3 5
1299484805.000000 10.0.0.1 20 10.0.0.2 1029
1299488395.000000 10.0.0.2 20 10.0.0.3 5
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-09-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -242,7 +242,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -256,17 +256,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299488e+09 10.0.0.1 20 10.0.0.2 1030
1.299492e+09 10.0.0.2 20 10.0.0.3 6
1299488405.000000 10.0.0.1 20 10.0.0.2 1030
1299491995.000000 10.0.0.2 20 10.0.0.3 6
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-10-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -279,7 +279,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -293,17 +293,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299492e+09 10.0.0.1 20 10.0.0.2 1031
1.299496e+09 10.0.0.2 20 10.0.0.3 7
1299492005.000000 10.0.0.1 20 10.0.0.2 1031
1299495595.000000 10.0.0.2 20 10.0.0.3 7
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-11-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -316,7 +316,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -330,17 +330,17 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299496e+09 10.0.0.1 20 10.0.0.2 1032
1.2995e+09 10.0.0.2 20 10.0.0.3 8
1299495605.000000 10.0.0.1 20 10.0.0.2 1032
1299499195.000000 10.0.0.2 20 10.0.0.3 8
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
> test.2011-03-07-12-00-05.ds
# Extent Types ...
<ExtentType name="DataSeries: ExtentIndex">
@ -353,7 +353,7 @@ offset extenttype
</ExtentType>
<ExtentType name="test" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -367,14 +367,14 @@ offset extenttype
extent offset ExtentType
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex
# Extent, type='test'
t id.orig_h id.orig_p id.resp_h id.resp_p
1.299499e+09 10.0.0.1 20 10.0.0.2 1033
1.299503e+09 10.0.0.2 20 10.0.0.3 9
1299499205.000000 10.0.0.1 20 10.0.0.2 1033
1299502795.000000 10.0.0.2 20 10.0.0.3 9
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
372 test
484 DataSeries: ExtentIndex
392 test
516 DataSeries: ExtentIndex

View file

@ -9,7 +9,7 @@
</ExtentType>
<ExtentType name="ssh" version="1.0" namespace="bro-ids.org">
<field type="double" name="t" pack_relative="t" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="t" pack_relative="t" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
<field type="variable32" name="id.resp_h" pack_unique="yes"/>
@ -27,17 +27,17 @@
extent offset ExtentType
40 DataSeries: XmlType
416 ssh
624 DataSeries: ExtentIndex
436 ssh
644 DataSeries: ExtentIndex
# Extent, type='ssh'
t id.orig_h id.orig_p id.resp_h id.resp_p status country
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 success unknown
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure US
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure UK
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 success BR
X.XXXe+09 1.2.3.4 1234 2.3.4.5 80 failure MX
1337058239.030366 1.2.3.4 1234 2.3.4.5 80 success unknown
1337058239.030366 1.2.3.4 1234 2.3.4.5 80 failure US
1337058239.030366 1.2.3.4 1234 2.3.4.5 80 failure UK
1337058239.030366 1.2.3.4 1234 2.3.4.5 80 success BR
1337058239.030366 1.2.3.4 1234 2.3.4.5 80 failure MX
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
416 ssh
624 DataSeries: ExtentIndex
436 ssh
644 DataSeries: ExtentIndex

View file

@ -9,7 +9,7 @@
</ExtentType>
<ExtentType name="conn" version="1.0" namespace="bro-ids.org">
<field type="double" name="ts" pack_relative="ts" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
@ -17,7 +17,7 @@
<field type="int64" name="id.resp_p" />
<field type="variable32" name="proto" pack_unique="yes"/>
<field type="variable32" name="service" pack_unique="yes"/>
<field type="double" name="duration" pack_relative="duration" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="duration" pack_relative="duration" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="int64" name="orig_bytes" />
<field type="int64" name="resp_bytes" />
<field type="variable32" name="conn_state" pack_unique="yes"/>
@ -51,46 +51,46 @@
extent offset ExtentType
40 DataSeries: XmlType
680 conn
2592 DataSeries: ExtentIndex
700 conn
2860 DataSeries: ExtentIndex
# Extent, type='conn'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
1.300475e+09 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0
1.300475e+09 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0
1.300475e+09 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0
1.300475e+09 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0 0 89 SHR F 0 Cd 0 0 1 117
1.300475e+09 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0 0 99 SHR F 0 Cd 0 0 1 127
1.300475e+09 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0 0 183 SHR F 0 Cd 0 0 1 211
1.300475e+09 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0 0 131 SHR F 0 Cd 0 0 1 159
1.300475e+09 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0 0 198 SHR F 0 Cd 0 0 1 226
1.300475e+09 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0
1.300475e+09 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 0 350 0 S0 F 0 D 7 546 0 0
1.300475e+09 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0
1.300475e+09 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0
1.300475e+09 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0 66 0 S0 F 0 D 2 162 0 0
1.300475e+09 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0 66 0 S0 F 0 D 2 122 0 0
1.300475e+09 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0
1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0 1130 734 S1 F 1130 ShACad 4 216 4 950
1.300475e+09 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0 0 350 OTH F 0 CdA 1 52 1 402
1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0 1178 734 S1 F 1178 ShACad 4 216 4 950
1.300475e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0 534 412 S1 F 534 ShACad 3 164 3 576
1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0 1148 734 S1 F 1148 ShACad 4 216 4 950
1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0 1171 733 S1 F 1171 ShACad 4 216 4 949
1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0 1137 733 S1 F 1137 ShACad 4 216 4 949
1.300475e+09 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48
1.300475e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0 525 232 S1 F 525 ShACad 3 164 3 396
1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0 1125 734 S1 F 1125 ShACad 4 216 4 950
1300475167.096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
1300475167.097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0
1300475167.099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0
1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 0 89 SHR F 0 Cd 0 0 1 117
1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 0 99 SHR F 0 Cd 0 0 1 127
1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 0 183 SHR F 0 Cd 0 0 1 211
1300475168.857956 3PKsZ2Uye21 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 0 89 SHR F 0 Cd 0 0 1 117
1300475168.858306 VW0XPVINV8a 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 0 99 SHR F 0 Cd 0 0 1 127
1300475168.858713 fRFu0wcOle6 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 0 183 SHR F 0 Cd 0 0 1 211
1300475168.891644 qSsw6ESzHV4 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 0 89 SHR F 0 Cd 0 0 1 117
1300475168.892037 iE6yhOq3SF 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 0 99 SHR F 0 Cd 0 0 1 127
1300475168.892414 GSxOnSLghOa 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 0 183 SHR F 0 Cd 0 0 1 211
1300475168.893988 qCaWGmzFtM5 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 0 89 SHR F 0 Cd 0 0 1 117
1300475168.894422 70MGiRM1Qf4 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 0 99 SHR F 0 Cd 0 0 1 127
1300475168.894787 h5DsfNtYzi1 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 0 183 SHR F 0 Cd 0 0 1 211
1300475168.901749 P654jzLoe3a 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 0 131 SHR F 0 Cd 0 0 1 159
1300475168.902195 Tw8jXtpTGu6 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 0 198 SHR F 0 Cd 0 0 1 226
1300475169.899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0
1300475170.862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0
1300475171.675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0
1300475171.677081 v5rgkJBig5l 141.142.220.226 55131 224.0.0.252 5355 udp dns 0.100021 66 0 S0 F 0 D 2 122 0 0
1300475173.116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0
1300475173.117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0
1300475173.153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0
1300475168.859163 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 tcp 0.215893 1130 734 S1 F 1130 ShACad 4 216 4 950
1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 0 350 OTH F 0 CdA 1 52 1 402
1300475168.895267 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 tcp 0.227284 1178 734 S1 F 1178 ShACad 4 216 4 950
1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0.120041 534 412 S1 F 534 ShACad 3 164 3 576
1300475168.892936 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 tcp 0.229603 1148 734 S1 F 1148 ShACad 4 216 4 950
1300475168.855305 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 tcp 0.218501 1171 733 S1 F 1171 ShACad 4 216 4 949
1300475168.892913 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 tcp 0.220961 1137 733 S1 F 1137 ShACad 4 216 4 949
1300475169.780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48
1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0.119905 525 232 S1 F 525 ShACad 3 164 3 396
1300475168.855330 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 tcp 0.219720 1125 734 S1 F 1125 ShACad 4 216 4 950
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
680 conn
2592 DataSeries: ExtentIndex
700 conn
2860 DataSeries: ExtentIndex

View file

@ -9,7 +9,7 @@
</ExtentType>
<ExtentType name="http" version="1.0" namespace="bro-ids.org">
<field type="double" name="ts" pack_relative="ts" pack_scale="1000" pack_scale_warn="no"/>
<field type="double" name="ts" pack_relative="ts" pack_scale="1e-6" print_format="%.6f" pack_scale_warn="no"/>
<field type="variable32" name="uid" pack_unique="yes"/>
<field type="variable32" name="id.orig_h" pack_unique="yes"/>
<field type="int64" name="id.orig_p" />
@ -65,26 +65,26 @@
extent offset ExtentType
40 DataSeries: XmlType
784 http
1172 DataSeries: ExtentIndex
804 http
1252 DataSeries: ExtentIndex
# Extent, type='http'
ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
1.300475e+09 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0
1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0
1.300475e+09 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1.300475e+09 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475168.843894 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0
1300475168.975800 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475168.976327 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475168.979160 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.012666 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.012730 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.014860 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.022665 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0
1300475169.036294 c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.036798 EAr0uf4mhq 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.039923 GvmoxJFXdTa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.074793 0Q4FH8sESw5 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.074938 slFea8xwSmb 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0
1300475169.075065 UfGkYA2HI2g 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0
# Extent, type='DataSeries: ExtentIndex'
offset extenttype
40 DataSeries: XmlType
784 http
1172 DataSeries: ExtentIndex
804 http
1252 DataSeries: ExtentIndex

View file

@ -3,7 +3,7 @@
# @TEST-GROUP: dataseries
#
# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
# @TEST-EXEC: ds2txt ssh.ds | ${SCRIPTS}/diff-remove-timestamps-dataseries >ssh.ds.txt
# @TEST-EXEC: ds2txt ssh.ds >ssh.ds.txt
# @TEST-EXEC: btest-diff ssh.ds.txt
module SSH;

View file

@ -1,6 +0,0 @@
#! /usr/bin/env bash
#
# Replace anything which looks like DataSeries timestamps (which is a double) with XXXs.
sed 's/1\.[0-9]*e+09/X.XXXe+09/g'