Test case for a dynamic input reader.

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-05-15-14-10-48
+#open	2014-07-31-19-06-48
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -14,6 +14,40 @@ scripts/base/init-bare.bro
   build/scripts/base/bif/reporter.bif.bro
   build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro
   build/scripts/base/bif/event.bif.bro
+  scripts/base/frameworks/logging/__load__.bro
+    scripts/base/frameworks/logging/main.bro
+      build/scripts/base/bif/logging.bif.bro
+    scripts/base/frameworks/logging/postprocessors/__load__.bro
+      scripts/base/frameworks/logging/postprocessors/scp.bro
+      scripts/base/frameworks/logging/postprocessors/sftp.bro
+    scripts/base/frameworks/logging/writers/ascii.bro
+    scripts/base/frameworks/logging/writers/dataseries.bro
+    scripts/base/frameworks/logging/writers/sqlite.bro
+    scripts/base/frameworks/logging/writers/elasticsearch.bro
+    scripts/base/frameworks/logging/writers/none.bro
+  scripts/base/frameworks/input/__load__.bro
+    scripts/base/frameworks/input/main.bro
+      build/scripts/base/bif/input.bif.bro
+    scripts/base/frameworks/input/readers/ascii.bro
+    scripts/base/frameworks/input/readers/raw.bro
+    scripts/base/frameworks/input/readers/benchmark.bro
+    scripts/base/frameworks/input/readers/binary.bro
+    scripts/base/frameworks/input/readers/sqlite.bro
+  scripts/base/frameworks/analyzer/__load__.bro
+    scripts/base/frameworks/analyzer/main.bro
+      scripts/base/frameworks/packet-filter/utils.bro
+      build/scripts/base/bif/analyzer.bif.bro
+  scripts/base/frameworks/files/__load__.bro
+    scripts/base/frameworks/files/main.bro
+      build/scripts/base/bif/file_analysis.bif.bro
+      scripts/base/utils/site.bro
+        scripts/base/utils/patterns.bro
+    scripts/base/frameworks/files/magic/__load__.bro
+  build/scripts/base/bif/__load__.bro
+    build/scripts/base/bif/broxygen.bif.bro
+    build/scripts/base/bif/bloom-filter.bif.bro
+    build/scripts/base/bif/cardinality-counter.bif.bro
+    build/scripts/base/bif/top-k.bif.bro
   build/scripts/base/bif/plugins/__load__.bro
     build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro
@@ -71,40 +105,16 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_X509.events.bif.bro
     build/scripts/base/bif/plugins/Bro_X509.types.bif.bro
     build/scripts/base/bif/plugins/Bro_X509.functions.bif.bro
-  scripts/base/frameworks/logging/__load__.bro
-    scripts/base/frameworks/logging/main.bro
-      build/scripts/base/bif/logging.bif.bro
-    scripts/base/frameworks/logging/postprocessors/__load__.bro
-      scripts/base/frameworks/logging/postprocessors/scp.bro
-      scripts/base/frameworks/logging/postprocessors/sftp.bro
-    scripts/base/frameworks/logging/writers/ascii.bro
-    scripts/base/frameworks/logging/writers/dataseries.bro
-    scripts/base/frameworks/logging/writers/sqlite.bro
-    scripts/base/frameworks/logging/writers/elasticsearch.bro
-    scripts/base/frameworks/logging/writers/none.bro
-  scripts/base/frameworks/input/__load__.bro
-    scripts/base/frameworks/input/main.bro
-      build/scripts/base/bif/input.bif.bro
-    scripts/base/frameworks/input/readers/ascii.bro
-    scripts/base/frameworks/input/readers/raw.bro
-    scripts/base/frameworks/input/readers/benchmark.bro
-    scripts/base/frameworks/input/readers/binary.bro
-    scripts/base/frameworks/input/readers/sqlite.bro
-  scripts/base/frameworks/analyzer/__load__.bro
-    scripts/base/frameworks/analyzer/main.bro
-      scripts/base/frameworks/packet-filter/utils.bro
-      build/scripts/base/bif/analyzer.bif.bro
-  scripts/base/frameworks/files/__load__.bro
-    scripts/base/frameworks/files/main.bro
-      build/scripts/base/bif/file_analysis.bif.bro
-      scripts/base/utils/site.bro
-        scripts/base/utils/patterns.bro
-    scripts/base/frameworks/files/magic/__load__.bro
-  build/scripts/base/bif/__load__.bro
-    build/scripts/base/bif/bloom-filter.bif.bro
-    build/scripts/base/bif/cardinality-counter.bif.bro
-    build/scripts/base/bif/top-k.bif.bro
-    build/scripts/base/bif/broxygen.bif.bro
+    build/scripts/base/bif/plugins/Bro_AsciiReader.ascii.bif.bro
+    build/scripts/base/bif/plugins/Bro_BenchmarkReader.benchmark.bif.bro
+    build/scripts/base/bif/plugins/Bro_BinaryReader.binary.bif.bro
+    build/scripts/base/bif/plugins/Bro_RawReader.raw.bif.bro
+    build/scripts/base/bif/plugins/Bro_SQLiteReader.sqlite.bif.bro
+    build/scripts/base/bif/plugins/Bro_AsciiWriter.ascii.bif.bro
+    build/scripts/base/bif/plugins/Bro_DataSeriesWriter.dataseries.bif.bro
+    build/scripts/base/bif/plugins/Bro_ElasticSearchWriter.elasticsearch.bif.bro
+    build/scripts/base/bif/plugins/Bro_NoneWriter.none.bif.bro
+    build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2014-05-15-14-10-48
+#close	2014-07-31-19-06-48

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-05-15-14-12-26
+#open	2014-07-31-19-07-23
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -14,6 +14,40 @@ scripts/base/init-bare.bro
   build/scripts/base/bif/reporter.bif.bro
   build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro
   build/scripts/base/bif/event.bif.bro
+  scripts/base/frameworks/logging/__load__.bro
+    scripts/base/frameworks/logging/main.bro
+      build/scripts/base/bif/logging.bif.bro
+    scripts/base/frameworks/logging/postprocessors/__load__.bro
+      scripts/base/frameworks/logging/postprocessors/scp.bro
+      scripts/base/frameworks/logging/postprocessors/sftp.bro
+    scripts/base/frameworks/logging/writers/ascii.bro
+    scripts/base/frameworks/logging/writers/dataseries.bro
+    scripts/base/frameworks/logging/writers/sqlite.bro
+    scripts/base/frameworks/logging/writers/elasticsearch.bro
+    scripts/base/frameworks/logging/writers/none.bro
+  scripts/base/frameworks/input/__load__.bro
+    scripts/base/frameworks/input/main.bro
+      build/scripts/base/bif/input.bif.bro
+    scripts/base/frameworks/input/readers/ascii.bro
+    scripts/base/frameworks/input/readers/raw.bro
+    scripts/base/frameworks/input/readers/benchmark.bro
+    scripts/base/frameworks/input/readers/binary.bro
+    scripts/base/frameworks/input/readers/sqlite.bro
+  scripts/base/frameworks/analyzer/__load__.bro
+    scripts/base/frameworks/analyzer/main.bro
+      scripts/base/frameworks/packet-filter/utils.bro
+      build/scripts/base/bif/analyzer.bif.bro
+  scripts/base/frameworks/files/__load__.bro
+    scripts/base/frameworks/files/main.bro
+      build/scripts/base/bif/file_analysis.bif.bro
+      scripts/base/utils/site.bro
+        scripts/base/utils/patterns.bro
+    scripts/base/frameworks/files/magic/__load__.bro
+  build/scripts/base/bif/__load__.bro
+    build/scripts/base/bif/broxygen.bif.bro
+    build/scripts/base/bif/bloom-filter.bif.bro
+    build/scripts/base/bif/cardinality-counter.bif.bro
+    build/scripts/base/bif/top-k.bif.bro
   build/scripts/base/bif/plugins/__load__.bro
     build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro
@@ -71,40 +105,16 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_X509.events.bif.bro
     build/scripts/base/bif/plugins/Bro_X509.types.bif.bro
     build/scripts/base/bif/plugins/Bro_X509.functions.bif.bro
-  scripts/base/frameworks/logging/__load__.bro
-    scripts/base/frameworks/logging/main.bro
-      build/scripts/base/bif/logging.bif.bro
-    scripts/base/frameworks/logging/postprocessors/__load__.bro
-      scripts/base/frameworks/logging/postprocessors/scp.bro
-      scripts/base/frameworks/logging/postprocessors/sftp.bro
-    scripts/base/frameworks/logging/writers/ascii.bro
-    scripts/base/frameworks/logging/writers/dataseries.bro
-    scripts/base/frameworks/logging/writers/sqlite.bro
-    scripts/base/frameworks/logging/writers/elasticsearch.bro
-    scripts/base/frameworks/logging/writers/none.bro
-  scripts/base/frameworks/input/__load__.bro
-    scripts/base/frameworks/input/main.bro
-      build/scripts/base/bif/input.bif.bro
-    scripts/base/frameworks/input/readers/ascii.bro
-    scripts/base/frameworks/input/readers/raw.bro
-    scripts/base/frameworks/input/readers/benchmark.bro
-    scripts/base/frameworks/input/readers/binary.bro
-    scripts/base/frameworks/input/readers/sqlite.bro
-  scripts/base/frameworks/analyzer/__load__.bro
-    scripts/base/frameworks/analyzer/main.bro
-      scripts/base/frameworks/packet-filter/utils.bro
-      build/scripts/base/bif/analyzer.bif.bro
-  scripts/base/frameworks/files/__load__.bro
-    scripts/base/frameworks/files/main.bro
-      build/scripts/base/bif/file_analysis.bif.bro
-      scripts/base/utils/site.bro
-        scripts/base/utils/patterns.bro
-    scripts/base/frameworks/files/magic/__load__.bro
-  build/scripts/base/bif/__load__.bro
-    build/scripts/base/bif/bloom-filter.bif.bro
-    build/scripts/base/bif/cardinality-counter.bif.bro
-    build/scripts/base/bif/top-k.bif.bro
-    build/scripts/base/bif/broxygen.bif.bro
+    build/scripts/base/bif/plugins/Bro_AsciiReader.ascii.bif.bro
+    build/scripts/base/bif/plugins/Bro_BenchmarkReader.benchmark.bif.bro
+    build/scripts/base/bif/plugins/Bro_BinaryReader.binary.bif.bro
+    build/scripts/base/bif/plugins/Bro_RawReader.raw.bif.bro
+    build/scripts/base/bif/plugins/Bro_SQLiteReader.sqlite.bif.bro
+    build/scripts/base/bif/plugins/Bro_AsciiWriter.ascii.bif.bro
+    build/scripts/base/bif/plugins/Bro_DataSeriesWriter.dataseries.bif.bro
+    build/scripts/base/bif/plugins/Bro_ElasticSearchWriter.elasticsearch.bif.bro
+    build/scripts/base/bif/plugins/Bro_NoneWriter.none.bif.bro
+    build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/base/init-default.bro
   scripts/base/utils/active-http.bro
     scripts/base/utils/exec.bro
@@ -236,4 +246,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2014-05-15-14-12-26
+#close	2014-07-31-19-07-23

testing/btest/Baseline/plugins.reader/out

@@ -0,0 +1,10 @@
+Input::EVENT_NEW
+^)kHV32-J_
+Input::EVENT_NEW
+(s[Q8J4Pu4
+Input::EVENT_NEW
++3iDbOB}kq
+Input::EVENT_NEW
+tz9dFehHz)
+Input::EVENT_NEW
+d&@3g)NljG

testing/btest/Baseline/plugins.reader/output

@@ -0,0 +1,4 @@
+Demo::Foo - A Foo test input reader (dynamic, version 1.0)
+    [Writer] Foo (Input::READER_FOO)
+
+===

testing/btest/plugins/reader-plugin/CMakeLists.txt

@@ -0,0 +1,17 @@
+
+project(Bro-Plugin-Demo-Foo)
+
+cmake_minimum_required(VERSION 2.6.3)
+
+if ( NOT BRO_DIST )
+    message(FATAL_ERROR "BRO_DIST not set")
+endif ()
+
+set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake)
+
+include(BroPlugin)
+
+bro_plugin_begin(Demo Foo)
+bro_plugin_cc(src/Plugin.cc)
+bro_plugin_cc(src/Foo.cc)
+bro_plugin_end()

testing/btest/plugins/reader-plugin/src/Foo.cc

@@ -0,0 +1,185 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "Foo.h"
+
+#include "threading/SerialTypes.h"
+#include "threading/Manager.h"
+
+using namespace input::reader;
+using threading::Value;
+using threading::Field;
+
+Foo::Foo(ReaderFrontend *frontend) : ReaderBackend(frontend)
+	{
+	ascii = new threading::formatter::Ascii(this, threading::formatter::Ascii::SeparatorInfo());
+	}
+
+Foo::~Foo()
+	{
+	DoClose();
+	delete ascii;
+	}
+
+void Foo::DoClose()
+	{
+	}
+
+bool Foo::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
+	{
+	DoUpdate();
+	return true;
+	}
+
+string Foo::RandomString(const int len)
+	{
+	string s(len, ' ');
+
+	static const char values[] =
+	"0123456789!@#$%^&*()-_=+{}[]\\|"
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	"abcdefghijklmnopqrstuvwxyz";
+
+	for (int i = 0; i < len; ++i)
+		s[i] = values[random() / (RAND_MAX / sizeof(values))];
+
+	return s;
+	}
+
+// read the entire file and send appropriate thingies back to InputMgr
+bool Foo::DoUpdate()
+	{
+	int linestosend = 5;
+	for ( int i = 0; i < linestosend; i++ )
+		{
+		Value** field = new Value*[NumFields()];
+		for  (int j = 0; j < NumFields(); j++ )
+			field[j] = EntryToVal(Fields()[j]->type, Fields()[j]->subtype);
+
+		SendEntry(field);
+	}
+
+	EndCurrentSend();
+
+	return true;
+}
+
+threading::Value* Foo::EntryToVal(TypeTag type, TypeTag subtype)
+	{
+	Value* val = new Value(type, true);
+
+	// basically construct something random from the fields that we want.
+
+	switch ( type ) {
+	case TYPE_ENUM:
+		assert(false); // no enums, please.
+
+	case TYPE_STRING:
+		{
+		string rnd = RandomString(10);
+		val->val.string_val.data = copy_string(rnd.c_str());
+		val->val.string_val.length = rnd.size();
+		break;
+		}
+
+	case TYPE_BOOL:
+		val->val.int_val = 1; // we never lie.
+		break;
+
+	case TYPE_INT:
+		val->val.int_val = random();
+		break;
+
+	case TYPE_TIME:
+		val->val.double_val = 0;
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_INTERVAL:
+		val->val.double_val = random();
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = random();
+		break;
+
+	case TYPE_PORT:
+		val->val.port_val.port = random() / (RAND_MAX / 60000);
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		break;
+
+	case TYPE_SUBNET:
+		{
+		val->val.subnet_val.prefix = ascii->ParseAddr("192.168.17.1");
+		val->val.subnet_val.length = 16;
+		}
+		break;
+
+	case TYPE_ADDR:
+		val->val.addr_val = ascii->ParseAddr("192.168.17.1");
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		// First - common initialization
+		// Then - initialization for table.
+		// Then - initialization for vector.
+		// Then - common stuff
+		{
+		// how many entries do we have...
+		unsigned int length = random() / (RAND_MAX / 15);
+
+		Value** lvals = new Value* [length];
+
+		if ( type == TYPE_TABLE )
+			{
+			val->val.set_val.vals = lvals;
+			val->val.set_val.size = length;
+			}
+		else if ( type == TYPE_VECTOR )
+			{
+			val->val.vector_val.vals = lvals;
+			val->val.vector_val.size = length;
+			}
+		else
+			assert(false);
+
+		if ( length == 0 )
+			break; //empty
+
+		for ( unsigned int pos = 0; pos < length; pos++ )
+			{
+			Value* newval = EntryToVal(subtype, TYPE_ENUM);
+			if ( newval == 0 )
+				{
+				Error("Error while reading set");
+				delete val;
+				return 0;
+				}
+			lvals[pos] = newval;
+			}
+
+		break;
+		}
+
+
+	default:
+		Error(Fmt("unsupported field format %d", type));
+		delete val;
+		return 0;
+	}
+
+	return val;
+
+	}
+
+
+bool Foo::DoHeartbeat(double network_time, double current_time)
+{
+	return true;
+}

testing/btest/plugins/reader-plugin/src/Foo.h

@@ -0,0 +1,34 @@
+
+#ifndef BRO_PLUGIN_DEMO_FOO_H
+#define BRO_PLUGIN_DEMO_FOO_H
+
+#include "input/ReaderBackend.h"
+#include "threading/formatters/Ascii.h"
+
+namespace input { namespace reader {
+
+/**
+ * A Foo reader to measure performance of the input framework.
+ */
+class Foo : public ReaderBackend {
+public:
+	Foo(ReaderFrontend* frontend);
+	~Foo();
+
+	static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Foo(frontend); }
+
+protected:
+	virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields);
+	virtual void DoClose();
+	virtual bool DoUpdate();
+	virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+	string RandomString(const int len);
+	threading::Value* EntryToVal(TypeTag Type, TypeTag subtype);
+	threading::formatter::Ascii* ascii;
+};
+
+} }
+
+#endif

testing/btest/plugins/reader-plugin/src/Plugin.cc

@@ -0,0 +1,19 @@
+#include "Plugin.h"
+
+#include "Foo.h"
+
+namespace plugin { namespace Demo_Foo { Plugin plugin; } }
+
+using namespace plugin::Demo_Foo;
+
+plugin::Configuration Plugin::Configure()
+	{
+	AddComponent(new ::input::Component("Foo", ::input::reader::Foo::Instantiate));
+
+	plugin::Configuration config;
+	config.name = "Demo::Foo";
+	config.description = "A Foo test input reader";
+	config.version.major = 1;
+	config.version.minor = 0;
+	return config;
+	}

testing/btest/plugins/reader-plugin/src/Plugin.h

@@ -0,0 +1,22 @@
+
+#ifndef BRO_PLUGIN_DEMO_FOO
+#define BRO_PLUGIN_DEMO_FOO
+
+#include <plugin/Plugin.h>
+
+namespace plugin {
+namespace Demo_Foo {
+
+class Plugin : public ::plugin::Plugin
+{
+protected:
+	// Overridden from plugin::Plugin.
+	virtual plugin::Configuration Configure();
+};
+
+extern Plugin plugin;
+
+}
+}
+
+#endif

testing/btest/plugins/reader.bro

@@ -0,0 +1,40 @@
+# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo
+# @TEST-EXEC: cp -r %DIR/reader-plugin/* .
+# @TEST-EXEC: make BRO=${DIST}
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output
+# @TEST-EXEC: echo === >>output
+# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` btest-bg-run bro bro %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff out
+
+redef exit_only_after_terminate = T;
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, tpe;
+	print outfile, s;
+	try = try + 1;
+	if ( try == 5 )
+		{
+		Input::remove("input");
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $reader=Input::READER_FOO, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	}