diff --git a/doc/logging-dataseries.rst b/doc/logging-dataseries.rst index e530ba7c0b..6eef223a90 100644 --- a/doc/logging-dataseries.rst +++ b/doc/logging-dataseries.rst @@ -109,22 +109,8 @@ TODO Warning, while packing field not_valid_after of record 11, error was > 10%: (1346460000 / 1000000 = 1346.46, round() = 1346) -* The compiler warn about a depracated method and I'm not immediately - seeing how to avoid using that. - * For testing our script-level options: - Can we get the extentsize from a ``.ds`` file? - Can we get the compressio level from a ``.ds`` file? -* ds2txt can apparently not read a file that is currently being - written. That's not good for the spool directory:: - - # ds2txt http.ds - **** Assertion failure in file - /DataSeriesSink.cpp, line 301 - **** Failed expression: tail[i] == 0xFF - **** Details: bad header for the tail of http.ds! - - Can that be worked around? - diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index 3c88c65653..aacef01f80 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -194,6 +194,8 @@ std::string DataSeries::GetDSOptionsForType(const threading::Field *field) if ( ! ds_use_integer_for_time ) s += " pack_scale=\"1000000\""; + else + s += string(" units=\"") + TIME_UNIT() + "\" epoch=\"unix\""; return s; } @@ -327,7 +329,13 @@ bool DataSeries::DoInit(string path, int num_fields, const threading::Field* con else Warning(Fmt("%s is not a valid compression type. Valid types are: 'lzf', 'lzo', 'gz', 'bz2', 'none', 'any'. Defaulting to 'any'", ds_compression.c_str())); - log_type = log_types.registerType(schema); + const ExtentType& type = log_types.registerTypeR(schema); + + // Note: This is a bit dicey as it depends on the implementation of + // registerTypeR(), but its what the DataSeries guys recommended + // given that we function we originally used has been deprecated. + log_type = &type; + log_series.setType(*log_type); return OpenLog(path); diff --git a/src/logging/writers/DataSeries.h b/src/logging/writers/DataSeries.h index bd2eb418f6..ab2bcec88c 100644 --- a/src/logging/writers/DataSeries.h +++ b/src/logging/writers/DataSeries.h @@ -43,6 +43,7 @@ private: static const size_t THREAD_MIN = 1; // Minimum number of compression threads that DataSeries may spawn. static const size_t THREAD_MAX = 128; // Maximum number of compression threads that DataSeries may spawn. static const size_t TIME_SCALE = 1000000; // Fixed-point multiplier for time values when converted to integers. + const char* TIME_UNIT() { return "microseconds"; } // DS name for time resolution when converted to integers. Must match TIME_SCALE. struct SchemaValue { diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out index b6f05003f3..a12fed36e1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out @@ -1,13 +1,13 @@ -test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 -test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 -test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 -test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 -test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 -test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 -test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 -test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 -test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 -test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 +test.2011-03-07-03-00-05.ds test 11-03-07_03.00.05 11-03-07_04.00.05 0 dataseries +test.2011-03-07-04-00-05.ds test 11-03-07_04.00.05 11-03-07_05.00.05 0 dataseries +test.2011-03-07-05-00-05.ds test 11-03-07_05.00.05 11-03-07_06.00.05 0 dataseries +test.2011-03-07-06-00-05.ds test 11-03-07_06.00.05 11-03-07_07.00.05 0 dataseries +test.2011-03-07-07-00-05.ds test 11-03-07_07.00.05 11-03-07_08.00.05 0 dataseries +test.2011-03-07-08-00-05.ds test 11-03-07_08.00.05 11-03-07_09.00.05 0 dataseries +test.2011-03-07-09-00-05.ds test 11-03-07_09.00.05 11-03-07_10.00.05 0 dataseries +test.2011-03-07-10-00-05.ds test 11-03-07_10.00.05 11-03-07_11.00.05 0 dataseries +test.2011-03-07-11-00-05.ds test 11-03-07_11.00.05 11-03-07_12.00.05 0 dataseries +test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataseries > test.2011-03-07-03-00-05.ds # Extent Types ... diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt index e6294b1d71..65d4ba0a67 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt @@ -9,7 +9,7 @@ - + @@ -17,7 +17,7 @@ - + @@ -51,8 +51,8 @@ extent offset ExtentType 40 DataSeries: XmlType -636 conn -2912 DataSeries: ExtentIndex +672 conn +2948 DataSeries: ExtentIndex # Extent, type='conn' ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes 1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 @@ -92,5 +92,5 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes # Extent, type='DataSeries: ExtentIndex' offset extenttype 40 DataSeries: XmlType -636 conn -2912 DataSeries: ExtentIndex +672 conn +2948 DataSeries: ExtentIndex diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro index 77ea32908a..fc3752a168 100644 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/options.bro @@ -1,5 +1,6 @@ # # @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries # # @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: test -e ssh.ds.xml diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro index 639c7f3562..6a0cee5888 100644 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro @@ -1,5 +1,6 @@ # # @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries # # @TEST-EXEC: bro -b -r %DIR/../rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out # @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt $i; done >>out diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro index 76f2451477..d04b0acf44 100644 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro @@ -1,5 +1,6 @@ # # @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries # # @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: ds2txt ssh.ds | ${SCRIPTS}/diff-remove-timestamps-dataseries >ssh.ds.txt diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro index 3a072998c0..e4dd6a5431 100644 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro @@ -1,5 +1,6 @@ # # @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries # # @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: ds2txt conn.ds >conn.ds.txt diff --git a/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro b/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro index 4a4b70afc2..38726a8b10 100644 --- a/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro +++ b/testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro @@ -1,5 +1,6 @@ # # @TEST-REQUIRES: has-writer DataSeries && which ds2txt +# @TEST-GROUP: dataseries # # @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES # @TEST-EXEC: ds2txt conn.ds >conn.ds.txt