From 3d1442e86b60d7bea4698b97a1dc91a75d3e70f4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 31 Jul 2014 12:04:27 -0700 Subject: [PATCH] Test case for a dynamic input reader. --- .../canonified_loaded_scripts.log | 82 ++++---- .../canonified_loaded_scripts.log | 82 ++++---- testing/btest/Baseline/plugins.reader/out | 10 + testing/btest/Baseline/plugins.reader/output | 4 + .../btest/plugins/reader-plugin/.btest-ignore | 0 .../plugins/reader-plugin/CMakeLists.txt | 17 ++ .../btest/plugins/reader-plugin/src/Foo.cc | 185 ++++++++++++++++++ testing/btest/plugins/reader-plugin/src/Foo.h | 34 ++++ .../btest/plugins/reader-plugin/src/Plugin.cc | 19 ++ .../btest/plugins/reader-plugin/src/Plugin.h | 22 +++ testing/btest/plugins/reader.bro | 40 ++++ 11 files changed, 423 insertions(+), 72 deletions(-) create mode 100644 testing/btest/Baseline/plugins.reader/out create mode 100644 testing/btest/Baseline/plugins.reader/output create mode 100644 testing/btest/plugins/reader-plugin/.btest-ignore create mode 100644 testing/btest/plugins/reader-plugin/CMakeLists.txt create mode 100644 testing/btest/plugins/reader-plugin/src/Foo.cc create mode 100644 testing/btest/plugins/reader-plugin/src/Foo.h create mode 100644 testing/btest/plugins/reader-plugin/src/Plugin.cc create mode 100644 testing/btest/plugins/reader-plugin/src/Plugin.h create mode 100644 testing/btest/plugins/reader.bro diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8128554281..c4a29ca44d 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-05-15-14-10-48 +#open 2014-07-31-19-06-48 #fields name #types string scripts/base/init-bare.bro @@ -14,6 +14,40 @@ scripts/base/init-bare.bro build/scripts/base/bif/reporter.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/event.bif.bro + scripts/base/frameworks/logging/__load__.bro + scripts/base/frameworks/logging/main.bro + build/scripts/base/bif/logging.bif.bro + scripts/base/frameworks/logging/postprocessors/__load__.bro + scripts/base/frameworks/logging/postprocessors/scp.bro + scripts/base/frameworks/logging/postprocessors/sftp.bro + scripts/base/frameworks/logging/writers/ascii.bro + scripts/base/frameworks/logging/writers/dataseries.bro + scripts/base/frameworks/logging/writers/sqlite.bro + scripts/base/frameworks/logging/writers/elasticsearch.bro + scripts/base/frameworks/logging/writers/none.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/main.bro + build/scripts/base/bif/input.bif.bro + scripts/base/frameworks/input/readers/ascii.bro + scripts/base/frameworks/input/readers/raw.bro + scripts/base/frameworks/input/readers/benchmark.bro + scripts/base/frameworks/input/readers/binary.bro + scripts/base/frameworks/input/readers/sqlite.bro + scripts/base/frameworks/analyzer/__load__.bro + scripts/base/frameworks/analyzer/main.bro + scripts/base/frameworks/packet-filter/utils.bro + build/scripts/base/bif/analyzer.bif.bro + scripts/base/frameworks/files/__load__.bro + scripts/base/frameworks/files/main.bro + build/scripts/base/bif/file_analysis.bif.bro + scripts/base/utils/site.bro + scripts/base/utils/patterns.bro + scripts/base/frameworks/files/magic/__load__.bro + build/scripts/base/bif/__load__.bro + build/scripts/base/bif/broxygen.bif.bro + build/scripts/base/bif/bloom-filter.bif.bro + build/scripts/base/bif/cardinality-counter.bif.bro + build/scripts/base/bif/top-k.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -71,40 +105,16 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_X509.events.bif.bro build/scripts/base/bif/plugins/Bro_X509.types.bif.bro build/scripts/base/bif/plugins/Bro_X509.functions.bif.bro - scripts/base/frameworks/logging/__load__.bro - scripts/base/frameworks/logging/main.bro - build/scripts/base/bif/logging.bif.bro - scripts/base/frameworks/logging/postprocessors/__load__.bro - scripts/base/frameworks/logging/postprocessors/scp.bro - scripts/base/frameworks/logging/postprocessors/sftp.bro - scripts/base/frameworks/logging/writers/ascii.bro - scripts/base/frameworks/logging/writers/dataseries.bro - scripts/base/frameworks/logging/writers/sqlite.bro - scripts/base/frameworks/logging/writers/elasticsearch.bro - scripts/base/frameworks/logging/writers/none.bro - scripts/base/frameworks/input/__load__.bro - scripts/base/frameworks/input/main.bro - build/scripts/base/bif/input.bif.bro - scripts/base/frameworks/input/readers/ascii.bro - scripts/base/frameworks/input/readers/raw.bro - scripts/base/frameworks/input/readers/benchmark.bro - scripts/base/frameworks/input/readers/binary.bro - scripts/base/frameworks/input/readers/sqlite.bro - scripts/base/frameworks/analyzer/__load__.bro - scripts/base/frameworks/analyzer/main.bro - scripts/base/frameworks/packet-filter/utils.bro - build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/files/__load__.bro - scripts/base/frameworks/files/main.bro - build/scripts/base/bif/file_analysis.bif.bro - scripts/base/utils/site.bro - scripts/base/utils/patterns.bro - scripts/base/frameworks/files/magic/__load__.bro - build/scripts/base/bif/__load__.bro - build/scripts/base/bif/bloom-filter.bif.bro - build/scripts/base/bif/cardinality-counter.bif.bro - build/scripts/base/bif/top-k.bif.bro - build/scripts/base/bif/broxygen.bif.bro + build/scripts/base/bif/plugins/Bro_AsciiReader.ascii.bif.bro + build/scripts/base/bif/plugins/Bro_BenchmarkReader.benchmark.bif.bro + build/scripts/base/bif/plugins/Bro_BinaryReader.binary.bif.bro + build/scripts/base/bif/plugins/Bro_RawReader.raw.bif.bro + build/scripts/base/bif/plugins/Bro_SQLiteReader.sqlite.bif.bro + build/scripts/base/bif/plugins/Bro_AsciiWriter.ascii.bif.bro + build/scripts/base/bif/plugins/Bro_DataSeriesWriter.dataseries.bif.bro + build/scripts/base/bif/plugins/Bro_ElasticSearchWriter.elasticsearch.bif.bro + build/scripts/base/bif/plugins/Bro_NoneWriter.none.bif.bro + build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2014-05-15-14-10-48 +#close 2014-07-31-19-06-48 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 03c299141c..661d58501a 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-05-15-14-12-26 +#open 2014-07-31-19-07-23 #fields name #types string scripts/base/init-bare.bro @@ -14,6 +14,40 @@ scripts/base/init-bare.bro build/scripts/base/bif/reporter.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/event.bif.bro + scripts/base/frameworks/logging/__load__.bro + scripts/base/frameworks/logging/main.bro + build/scripts/base/bif/logging.bif.bro + scripts/base/frameworks/logging/postprocessors/__load__.bro + scripts/base/frameworks/logging/postprocessors/scp.bro + scripts/base/frameworks/logging/postprocessors/sftp.bro + scripts/base/frameworks/logging/writers/ascii.bro + scripts/base/frameworks/logging/writers/dataseries.bro + scripts/base/frameworks/logging/writers/sqlite.bro + scripts/base/frameworks/logging/writers/elasticsearch.bro + scripts/base/frameworks/logging/writers/none.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/main.bro + build/scripts/base/bif/input.bif.bro + scripts/base/frameworks/input/readers/ascii.bro + scripts/base/frameworks/input/readers/raw.bro + scripts/base/frameworks/input/readers/benchmark.bro + scripts/base/frameworks/input/readers/binary.bro + scripts/base/frameworks/input/readers/sqlite.bro + scripts/base/frameworks/analyzer/__load__.bro + scripts/base/frameworks/analyzer/main.bro + scripts/base/frameworks/packet-filter/utils.bro + build/scripts/base/bif/analyzer.bif.bro + scripts/base/frameworks/files/__load__.bro + scripts/base/frameworks/files/main.bro + build/scripts/base/bif/file_analysis.bif.bro + scripts/base/utils/site.bro + scripts/base/utils/patterns.bro + scripts/base/frameworks/files/magic/__load__.bro + build/scripts/base/bif/__load__.bro + build/scripts/base/bif/broxygen.bif.bro + build/scripts/base/bif/bloom-filter.bif.bro + build/scripts/base/bif/cardinality-counter.bif.bro + build/scripts/base/bif/top-k.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -71,40 +105,16 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_X509.events.bif.bro build/scripts/base/bif/plugins/Bro_X509.types.bif.bro build/scripts/base/bif/plugins/Bro_X509.functions.bif.bro - scripts/base/frameworks/logging/__load__.bro - scripts/base/frameworks/logging/main.bro - build/scripts/base/bif/logging.bif.bro - scripts/base/frameworks/logging/postprocessors/__load__.bro - scripts/base/frameworks/logging/postprocessors/scp.bro - scripts/base/frameworks/logging/postprocessors/sftp.bro - scripts/base/frameworks/logging/writers/ascii.bro - scripts/base/frameworks/logging/writers/dataseries.bro - scripts/base/frameworks/logging/writers/sqlite.bro - scripts/base/frameworks/logging/writers/elasticsearch.bro - scripts/base/frameworks/logging/writers/none.bro - scripts/base/frameworks/input/__load__.bro - scripts/base/frameworks/input/main.bro - build/scripts/base/bif/input.bif.bro - scripts/base/frameworks/input/readers/ascii.bro - scripts/base/frameworks/input/readers/raw.bro - scripts/base/frameworks/input/readers/benchmark.bro - scripts/base/frameworks/input/readers/binary.bro - scripts/base/frameworks/input/readers/sqlite.bro - scripts/base/frameworks/analyzer/__load__.bro - scripts/base/frameworks/analyzer/main.bro - scripts/base/frameworks/packet-filter/utils.bro - build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/files/__load__.bro - scripts/base/frameworks/files/main.bro - build/scripts/base/bif/file_analysis.bif.bro - scripts/base/utils/site.bro - scripts/base/utils/patterns.bro - scripts/base/frameworks/files/magic/__load__.bro - build/scripts/base/bif/__load__.bro - build/scripts/base/bif/bloom-filter.bif.bro - build/scripts/base/bif/cardinality-counter.bif.bro - build/scripts/base/bif/top-k.bif.bro - build/scripts/base/bif/broxygen.bif.bro + build/scripts/base/bif/plugins/Bro_AsciiReader.ascii.bif.bro + build/scripts/base/bif/plugins/Bro_BenchmarkReader.benchmark.bif.bro + build/scripts/base/bif/plugins/Bro_BinaryReader.binary.bif.bro + build/scripts/base/bif/plugins/Bro_RawReader.raw.bif.bro + build/scripts/base/bif/plugins/Bro_SQLiteReader.sqlite.bif.bro + build/scripts/base/bif/plugins/Bro_AsciiWriter.ascii.bif.bro + build/scripts/base/bif/plugins/Bro_DataSeriesWriter.dataseries.bif.bro + build/scripts/base/bif/plugins/Bro_ElasticSearchWriter.elasticsearch.bif.bro + build/scripts/base/bif/plugins/Bro_NoneWriter.none.bif.bro + build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/base/init-default.bro scripts/base/utils/active-http.bro scripts/base/utils/exec.bro @@ -236,4 +246,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2014-05-15-14-12-26 +#close 2014-07-31-19-07-23 diff --git a/testing/btest/Baseline/plugins.reader/out b/testing/btest/Baseline/plugins.reader/out new file mode 100644 index 0000000000..9dd3101a8d --- /dev/null +++ b/testing/btest/Baseline/plugins.reader/out @@ -0,0 +1,10 @@ +Input::EVENT_NEW +^)kHV32-J_ +Input::EVENT_NEW +(s[Q8J4Pu4 +Input::EVENT_NEW ++3iDbOB}kq +Input::EVENT_NEW +tz9dFehHz) +Input::EVENT_NEW +d&@3g)NljG diff --git a/testing/btest/Baseline/plugins.reader/output b/testing/btest/Baseline/plugins.reader/output new file mode 100644 index 0000000000..fa218d04a5 --- /dev/null +++ b/testing/btest/Baseline/plugins.reader/output @@ -0,0 +1,4 @@ +Demo::Foo - A Foo test input reader (dynamic, version 1.0) + [Writer] Foo (Input::READER_FOO) + +=== diff --git a/testing/btest/plugins/reader-plugin/.btest-ignore b/testing/btest/plugins/reader-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/reader-plugin/CMakeLists.txt b/testing/btest/plugins/reader-plugin/CMakeLists.txt new file mode 100644 index 0000000000..2234907ad2 --- /dev/null +++ b/testing/btest/plugins/reader-plugin/CMakeLists.txt @@ -0,0 +1,17 @@ + +project(Bro-Plugin-Demo-Foo) + +cmake_minimum_required(VERSION 2.6.3) + +if ( NOT BRO_DIST ) + message(FATAL_ERROR "BRO_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) + +include(BroPlugin) + +bro_plugin_begin(Demo Foo) +bro_plugin_cc(src/Plugin.cc) +bro_plugin_cc(src/Foo.cc) +bro_plugin_end() diff --git a/testing/btest/plugins/reader-plugin/src/Foo.cc b/testing/btest/plugins/reader-plugin/src/Foo.cc new file mode 100644 index 0000000000..9d79361068 --- /dev/null +++ b/testing/btest/plugins/reader-plugin/src/Foo.cc @@ -0,0 +1,185 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include +#include +#include + +#include "Foo.h" + +#include "threading/SerialTypes.h" +#include "threading/Manager.h" + +using namespace input::reader; +using threading::Value; +using threading::Field; + +Foo::Foo(ReaderFrontend *frontend) : ReaderBackend(frontend) + { + ascii = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo()); + } + +Foo::~Foo() + { + DoClose(); + delete ascii; + } + +void Foo::DoClose() + { + } + +bool Foo::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields) + { + DoUpdate(); + return true; + } + +string Foo::RandomString(const int len) + { + string s(len, ' '); + + static const char values[] = + "0123456789!@#$%^&*()-_=+{}[]\\|" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + for (int i = 0; i < len; ++i) + s[i] = values[random() / (RAND_MAX / sizeof(values))]; + + return s; + } + +// read the entire file and send appropriate thingies back to InputMgr +bool Foo::DoUpdate() + { + int linestosend = 5; + for ( int i = 0; i < linestosend; i++ ) + { + Value** field = new Value*[NumFields()]; + for (int j = 0; j < NumFields(); j++ ) + field[j] = EntryToVal(Fields()[j]->type, Fields()[j]->subtype); + + SendEntry(field); + } + + EndCurrentSend(); + + return true; +} + +threading::Value* Foo::EntryToVal(TypeTag type, TypeTag subtype) + { + Value* val = new Value(type, true); + + // basically construct something random from the fields that we want. + + switch ( type ) { + case TYPE_ENUM: + assert(false); // no enums, please. + + case TYPE_STRING: + { + string rnd = RandomString(10); + val->val.string_val.data = copy_string(rnd.c_str()); + val->val.string_val.length = rnd.size(); + break; + } + + case TYPE_BOOL: + val->val.int_val = 1; // we never lie. + break; + + case TYPE_INT: + val->val.int_val = random(); + break; + + case TYPE_TIME: + val->val.double_val = 0; + break; + + case TYPE_DOUBLE: + case TYPE_INTERVAL: + val->val.double_val = random(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = random(); + break; + + case TYPE_PORT: + val->val.port_val.port = random() / (RAND_MAX / 60000); + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: + { + val->val.subnet_val.prefix = ascii->ParseAddr("192.168.17.1"); + val->val.subnet_val.length = 16; + } + break; + + case TYPE_ADDR: + val->val.addr_val = ascii->ParseAddr("192.168.17.1"); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = random() / (RAND_MAX / 15); + + Value** lvals = new Value* [length]; + + if ( type == TYPE_TABLE ) + { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } + else if ( type == TYPE_VECTOR ) + { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } + else + assert(false); + + if ( length == 0 ) + break; //empty + + for ( unsigned int pos = 0; pos < length; pos++ ) + { + Value* newval = EntryToVal(subtype, TYPE_ENUM); + if ( newval == 0 ) + { + Error("Error while reading set"); + delete val; + return 0; + } + lvals[pos] = newval; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d", type)); + delete val; + return 0; + } + + return val; + + } + + +bool Foo::DoHeartbeat(double network_time, double current_time) +{ + return true; +} diff --git a/testing/btest/plugins/reader-plugin/src/Foo.h b/testing/btest/plugins/reader-plugin/src/Foo.h new file mode 100644 index 0000000000..490b209191 --- /dev/null +++ b/testing/btest/plugins/reader-plugin/src/Foo.h @@ -0,0 +1,34 @@ + +#ifndef BRO_PLUGIN_DEMO_FOO_H +#define BRO_PLUGIN_DEMO_FOO_H + +#include "input/ReaderBackend.h" +#include "threading/formatters/Ascii.h" + +namespace input { namespace reader { + +/** + * A Foo reader to measure performance of the input framework. + */ +class Foo : public ReaderBackend { +public: + Foo(ReaderFrontend* frontend); + ~Foo(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Foo(frontend); } + +protected: + virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields); + virtual void DoClose(); + virtual bool DoUpdate(); + virtual bool DoHeartbeat(double network_time, double current_time); + +private: + string RandomString(const int len); + threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); + threading::formatter::Ascii* ascii; +}; + +} } + +#endif diff --git a/testing/btest/plugins/reader-plugin/src/Plugin.cc b/testing/btest/plugins/reader-plugin/src/Plugin.cc new file mode 100644 index 0000000000..acc715511a --- /dev/null +++ b/testing/btest/plugins/reader-plugin/src/Plugin.cc @@ -0,0 +1,19 @@ +#include "Plugin.h" + +#include "Foo.h" + +namespace plugin { namespace Demo_Foo { Plugin plugin; } } + +using namespace plugin::Demo_Foo; + +plugin::Configuration Plugin::Configure() + { + AddComponent(new ::input::Component("Foo", ::input::reader::Foo::Instantiate)); + + plugin::Configuration config; + config.name = "Demo::Foo"; + config.description = "A Foo test input reader"; + config.version.major = 1; + config.version.minor = 0; + return config; + } diff --git a/testing/btest/plugins/reader-plugin/src/Plugin.h b/testing/btest/plugins/reader-plugin/src/Plugin.h new file mode 100644 index 0000000000..c65eac01a1 --- /dev/null +++ b/testing/btest/plugins/reader-plugin/src/Plugin.h @@ -0,0 +1,22 @@ + +#ifndef BRO_PLUGIN_DEMO_FOO +#define BRO_PLUGIN_DEMO_FOO + +#include + +namespace plugin { +namespace Demo_Foo { + +class Plugin : public ::plugin::Plugin +{ +protected: + // Overridden from plugin::Plugin. + virtual plugin::Configuration Configure(); +}; + +extern Plugin plugin; + +} +} + +#endif diff --git a/testing/btest/plugins/reader.bro b/testing/btest/plugins/reader.bro new file mode 100644 index 0000000000..cecb5306da --- /dev/null +++ b/testing/btest/plugins/reader.bro @@ -0,0 +1,40 @@ +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: cp -r %DIR/reader-plugin/* . +# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` btest-bg-run bro bro %INPUT +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff out + +redef exit_only_after_terminate = T; + +global outfile: file; +global try: count; + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + print outfile, tpe; + print outfile, s; + try = try + 1; + if ( try == 5 ) + { + Input::remove("input"); + close(outfile); + terminate(); + } + } + +event bro_init() + { + try = 0; + outfile = open("../out"); + Input::add_event([$source="../input.log", $reader=Input::READER_FOO, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]); + }