Checkpoint - all src/ except src/input

doc/index.rst

@@ -24,6 +24,7 @@ Frameworks
 
    notice
    logging
+   input
    cluster
    signatures
 

doc/input.rst

@@ -0,0 +1,183 @@
+=====================
+Loading Data into Bro
+=====================
+
+.. rst-class:: opening
+
+   Bro comes with a flexible input interface that allows to read
+   previously stored data. Data is either read into bro tables or
+   sent to scripts using events.
+   This document describes how the input framework can be used.
+
+.. contents::
+
+Terminology
+===========
+
+Bro's input framework is built around three main abstracts, that are 
+very similar to the abstracts used in the logging framework:
+
+        Input Streams
+                An input stream corresponds to a single input source 
+                (usually a textfile). It defined the information necessary
+                to find the source (e.g. the filename), the reader that it used
+                to get data from it (see below). 
+                It also defines exactly what data is read from the input source.
+                There are two different kind of streams, event streams and table 
+                streams.
+                By default, event streams generate an event for each line read
+                from the input source.
+                Table streams on the other hand read the input source in a bro
+                table for easy later access.
+
+        Readers
+                A reader defines the input format for the specific input stream.
+                At the moment, Bro comes with two types of reader. The default reader is READER_ASCII,
+                which can read the tab seperated ASCII logfiles that were generated by the
+                logging framework.
+                READER_RAW can files containing records separated by a character(like e.g. newline) and send
+                one event per line.
+
+
+Event Streams
+=============
+
+For examples, please look at the unit tests in
+``testing/btest/scripts/base/frameworks/input/``.
+
+Event Streams are streams that generate an event for each line in of the input source.
+
+For example, a simple stream retrieving the fields ``i`` and ``b`` from an inputSource
+could be defined as follows:
+
+.. code:: bro
+
+        type Val: record {
+	        i: int;
+                b: bool;
+        };
+
+        event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) {
+                # work with event data
+        }
+        
+        event bro_init {
+                Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]);
+        }
+
+The fields that can be set for an event stream are:
+
+        ``want_record``
+                Boolean value, that defines if the event wants to receive the fields inside of
+                a single record value, or individually (default).
+
+        ``source``
+                A mandatory string identifying the source of the data.
+                For the ASCII reader this is the filename.
+
+        ``reader``
+                The reader used for this stream. Default is ``READER_ASCII``.
+
+        ``mode``
+                The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``.
+                Default is ``MANUAL``.
+                ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not
+                be reflected in the data bro knows.
+                ``REREAD`` means that the whole file is read again each time a change is found. This should be used for 
+                files that are mapped to a table where individual lines can change.
+                ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new
+                data is added to the file.
+
+        ``name``
+                A mandatory name for the stream that can later be used
+                to remove it.
+
+        ``fields``
+                Name of a record type containing the fields, which should be retrieved from
+                the input stream.
+
+        ``ev``
+                The event which is fired, after a line has been read from the input source.
+                The first argument that is passed to the event is an Input::Event structure, 
+                followed by the data, either inside of a record (if ``want_record is set``) or as 
+                individual fields.
+                The Input::Event structure can contain information, if the received line is ``NEW``, has
+                been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information
+                for event filters, the value is always ``NEW`` at the moment.
+
+
+
+Table Streams
+=============
+
+Table streams are the second, more complex type of input streams.
+
+Table streams store the information they read from an input source in a bro table. For example,
+when reading a file that contains ip addresses and connection attemt information one could use
+an approach similar to this:
+
+.. code:: bro
+
+        type Idx: record {
+                a: addr;
+        };
+
+        type Val: record {
+                tries: count;
+        };
+
+        global conn_attempts: table[addr] of count = table();
+
+        event bro_init {
+                Input::add_table([$source="input.txt", $name="input", $idx=Idx, $val=Val, $destination=conn_attempts]);
+        }
+
+The table conn_attempts will then contain the information about connection attemps.
+
+The possible fields that can be set for an table stream are:
+
+        ``want_record``
+                Boolean value, that defines if the event wants to receive the fields inside of
+                a single record value, or individually (default).
+
+        ``source``
+                A mandatory string identifying the source of the data.
+                For the ASCII reader this is the filename.
+
+        ``reader``
+                The reader used for this stream. Default is ``READER_ASCII``.
+
+        ``mode``
+                The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``.
+                Default is ``MANUAL``.
+                ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not
+                be reflected in the data bro knows.
+                ``REREAD`` means that the whole file is read again each time a change is found. This should be used for 
+                files that are mapped to a table where individual lines can change.
+                ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new
+                data is added to the file.
+
+        ``name``
+                A mandatory name for the filter that can later be used
+                to manipulate it further.
+        
+        ``idx``
+                Record type that defines the index of the table
+
+        ``val``
+                Record type that defines the values of the table
+
+        ``want_record``
+                Defines if the values of the table should be stored as a record (default),
+                or as a simple value. Has to be set if Val contains more than one element.
+
+        ``destination``
+                The destination table
+
+        ``ev``
+                Optional event that is raised, when values are added to, changed in or deleted from the table.
+                Events are passed an Input::Event description as the first argument, the index record as the second argument
+                and the values as the third argument.
+
+        ``pred``
+                Optional predicate, that can prevent entries from being added to the table and events from being sent.

doc/scripts/DocSourcesList.cmake

@@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal)
 rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
@@ -31,6 +32,8 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
 rest_target(${psd} base/frameworks/communication/main.bro)
 rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
+rest_target(${psd} base/frameworks/input/main.bro)
+rest_target(${psd} base/frameworks/input/readers/ascii.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)

scripts/base/frameworks/input/__load__.bro

@@ -0,0 +1,5 @@
+@load ./main
+@load ./readers/ascii
+@load ./readers/raw
+@load ./readers/benchmark
+

scripts/base/frameworks/input/main.bro

@@ -0,0 +1,137 @@
+##! The input framework provides a way to read previously stored data either
+##! as an event stream or into a bro table.
+
+module Input;
+
+export {
+         
+	## The default input reader used. Defaults to `READER_ASCII`.
+	const default_reader = READER_ASCII &redef;
+
+	const default_mode = MANUAL &redef;
+
+	## TableFilter description type used for the `table` method.
+	type TableDescription: record {
+		## Common definitions for tables and events
+		
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+		
+		## Reader to use for this steam
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a filter at a later time
+		name: string; 
+
+		## Special definitions for tables
+
+		## Table which will contain the data read by the input framework
+		destination: any;
+		## Record that defines the values used as the index of the table
+		idx: any;
+		## Record that defines the values used as the values of the table
+		## If val is undefined, destination has to be a set.
+		val: any &optional;
+		## Defines if the value of the table is a record (default), or a single value.
+		## Val can only contain one element when this is set to false.
+		want_record: bool &default=T;
+
+		## The event that is raised each time a value is added to, changed in or removed from the table.
+		## The event will receive an Input::Event enum as the first argument, the idx record as the second argument
+		## and the value (record) as the third argument.
+		ev: any &optional; # event containing idx, val as values.		
+
+		## Predicate function, that can decide if an insertion, update or removal should really be executed.
+		## Parameters are the same as for the event. If true is returned, the update is performed. If false
+		## is returned, it is skipped
+		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
+	};
+
+	## EventFilter description type used for the `event` method.
+	type EventDescription: record {
+		## Common definitions for tables and events
+		
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+		
+		## Reader to use for this steam
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a filter at a later time
+		name: string; 
+
+		## Special definitions for events
+	
+		## Record describing the fields to be retrieved from the source input.
+		fields: any;
+		## If want_record if false (default), the event receives each value in fields as a seperate argument.
+		## If it is set to true, the event receives all fields in a signle record value.
+		want_record: bool &default=F;
+
+		## The event that is rised each time a new line is received from the reader.
+		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
+		ev: any; 
+
+	};
+
+	## Create a new table input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_table: function(description: Input::TableDescription) : bool;
+	
+	## Create a new event input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_event: function(description: Input::EventDescription) : bool;
+
+	## Remove a input stream. Returns true on success and false if the named stream was not found.
+	##
+	## id: string value identifying the stream to be removed
+	global remove: function(id: string) : bool;
+
+	## Forces the current input to be checked for changes.
+	## Returns true on success and false if the named stream was not found
+	##
+	## id: string value identifying the stream
+	global force_update: function(id: string) : bool;
+
+	## Event that is called, when the update of a specific source is finished
+	global update_finished: event(name: string, source:string);
+}
+
+@load base/input.bif
+
+
+module Input;
+
+#global streams: table[string] of Filter;
+# ^ change to set containing the names
+
+function add_table(description: Input::TableDescription) : bool
+	{
+	return __create_table_stream(description);
+	}
+
+function add_event(description: Input::EventDescription) : bool
+	{
+	return __create_event_stream(description);
+	}
+
+function remove(id: string) : bool
+	{
+	return __remove_stream(id);
+	}
+
+function force_update(id: string) : bool
+	{
+	return __force_update(id);
+	}
+

scripts/base/frameworks/input/readers/ascii.bro

@@ -0,0 +1,19 @@
+##! Interface for the ascii input reader.
+
+module InputAscii;
+
+export {
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = "-" &redef;
+}

scripts/base/frameworks/input/readers/benchmark.bro

@@ -0,0 +1,23 @@
+##! Interface for the ascii input reader.
+
+module InputBenchmark;
+
+export {
+	## multiplication factor for each second
+	const factor = 1.0 &redef;
+
+	## spread factor between lines
+	const spread = 0 &redef;
+
+	## spreading where usleep = 1000000 / autospread * num_lines
+	const autospread = 0.0 &redef;
+
+	## addition factor for each heartbeat
+	const addfactor = 0 &redef;
+
+	## stop spreading at x lines per heartbeat
+	const stopspreadat = 0 &redef;
+
+	## 1 -> enable timed spreading
+	const timedspread = 0.0 &redef;
+}

scripts/base/frameworks/input/readers/raw.bro

@@ -0,0 +1,9 @@
+##! Interface for the raw input reader.
+
+module InputRaw;
+
+export {
+	## Separator between input records.
+	## Please note that the separator has to be exactly one character long
+	const record_separator = "\n" &redef;
+}

scripts/base/init-bare.bro

@@ -1805,6 +1805,14 @@ export {
 	};
 } # end export
 
+module Threading;
+
+export {
+	## The heart beat interval used by the threading framework.
+	## Changing this should usually not be neccessary and will break several tests.
+	const heart_beat_interval = 1.0 secs &redef;
+}
+
 module GLOBAL;
 
 ## An NTP message.
@@ -2632,3 +2640,6 @@ const snaplen = 8192 &redef;
 # Load the logging framework here because it uses fairly deep integration with
 # BiFs and script-land defined types.
 @load base/frameworks/logging
+
+@load base/frameworks/input
+

src/Attr.cc

@@ -17,7 +17,7 @@ const char* attr_name(attr_tag t)
 		"&persistent", "&synchronized", "&postprocessor",
 		"&encrypt", "&match", "&disable_print_hook",
 		"&raw_output", "&mergeable", "&priority",
-		"&group", "&log", "&error_handler", "(&tracked)",
+		"&group", "&log", "&error_handler", "&type_column", "(&tracked)",
 	};
 
 	return attr_names[int(t)];
@@ -420,6 +420,25 @@ void Attributes::CheckAttr(Attr* a)
 			Error("&log applied to a type that cannot be logged");
 		break;
 
+	case ATTR_TYPE_COLUMN:
+		{
+		if ( type->Tag() != TYPE_PORT )
+			{
+			Error("type_column tag only applicable to ports");
+			break;
+			}
+
+		BroType* atype = a->AttrExpr()->Type();
+
+		if ( atype->Tag() != TYPE_STRING ) {
+			Error("type column needs to have a string argument");
+			break;
+		}
+
+		break;
+		}
+
+
 	default:
 		BadTag("Attributes::CheckAttr", attr_name(a->Tag()));
 	}

src/Attr.h

@@ -35,6 +35,7 @@ typedef enum {
 	ATTR_GROUP,
 	ATTR_LOG,
 	ATTR_ERROR_HANDLER,
+	ATTR_TYPE_COLUMN,	// for input framework
 	ATTR_TRACKED,	// hidden attribute, tracked by NotifierRegistry
 #define NUM_ATTRS (int(ATTR_TRACKED) + 1)
 } attr_tag;

src/CMakeLists.txt

@@ -142,6 +142,7 @@ endmacro(GET_BIF_OUTPUT_FILES)
 set(BIF_SRCS
     bro.bif
     logging.bif
+    input.bif
     event.bif
     const.bif
     types.bif
@@ -420,6 +421,13 @@ set(bro_SRCS
     logging/writers/DataSeries.cc
     logging/writers/None.cc
 
+    input/Manager.cc
+    input/ReaderBackend.cc
+    input/ReaderFrontend.cc
+    input/readers/Ascii.cc
+    input/readers/Raw.cc
+    input/readers/Benchmark.cc
+
     nb_dns.c
     digest.h
 )

src/DebugLogger.cc

@@ -15,7 +15,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
-	{ "logging", 0, false }, { "threading", 0, false }
+	{ "logging", 0, false }, {"input", 0, false }, 
+	{ "threading", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -24,6 +24,7 @@ enum DebugStream {
 	DBG_DPD,	// Dynamic application detection framework
 	DBG_TM,		// Time-machine packet input via Brocolli
 	DBG_LOGGING,	// Logging streams
+	DBG_INPUT,	// Input streams
 	DBG_THREADING,	// Threading system
 
 	NUM_DBGS // Has to be last

src/Func.cc

@@ -522,11 +522,13 @@ void builtin_error(const char* msg, BroObj* arg)
 
 #include "bro.bif.func_h"
 #include "logging.bif.func_h"
+#include "input.bif.func_h"
 #include "reporter.bif.func_h"
 #include "strings.bif.func_h"
 
 #include "bro.bif.func_def"
 #include "logging.bif.func_def"
+#include "input.bif.func_def"
 #include "reporter.bif.func_def"
 #include "strings.bif.func_def"
 
@@ -541,6 +543,7 @@ void init_builtin_funcs()
 
 #include "bro.bif.func_init"
 #include "logging.bif.func_init"
+#include "input.bif.func_init"
 #include "reporter.bif.func_init"
 #include "strings.bif.func_init"
 

src/NetVar.cc

@@ -242,6 +242,7 @@ StringVal* cmd_line_bpf_filter;
 #include "types.bif.netvar_def"
 #include "event.bif.netvar_def"
 #include "logging.bif.netvar_def"
+#include "input.bif.netvar_def"
 #include "reporter.bif.netvar_def"
 
 void init_event_handlers()
@@ -303,6 +304,7 @@ void init_net_var()
 #include "const.bif.netvar_init"
 #include "types.bif.netvar_init"
 #include "logging.bif.netvar_init"
+#include "input.bif.netvar_init"
 #include "reporter.bif.netvar_init"
 
 	conn_id = internal_type("conn_id")->AsRecordType();

src/NetVar.h

@@ -252,6 +252,7 @@ extern void init_net_var();
 #include "types.bif.netvar_h"
 #include "event.bif.netvar_h"
 #include "logging.bif.netvar_h"
+#include "input.bif.netvar_h"
 #include "reporter.bif.netvar_h"
 
 #endif

src/Val.cc

@@ -948,6 +948,11 @@ SubNetVal::SubNetVal(const IPAddr& addr, int width) : Val(TYPE_SUBNET)
 	val.subnet_val = new IPPrefix(addr, width);
 	}
 
+SubNetVal::SubNetVal(const IPPrefix& prefix) : Val(TYPE_SUBNET)
+	{
+	val.subnet_val = new IPPrefix(prefix);
+	}
+
 SubNetVal::~SubNetVal()
 	{
 	delete val.subnet_val;

src/Val.h

@@ -580,6 +580,7 @@ public:
 	SubNetVal(uint32 addr, int width); // IPv4.
 	SubNetVal(const uint32 addr[4], int width); // IPv6.
 	SubNetVal(const IPAddr& addr, int width);
+	SubNetVal(const IPPrefix& prefix);
 	~SubNetVal();
 
 	Val* SizeVal() const;
@@ -839,6 +840,9 @@ public:
 			timer = 0;
 		}
 
+	HashKey* ComputeHash(const Val* index) const
+		{ return table_hash->ComputeHash(index, 1); }
+
 protected:
 	friend class Val;
 	friend class StateAccess;
@@ -849,8 +853,6 @@ protected:
 	void CheckExpireAttr(attr_tag at);
 	int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val);
 	int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN);
-	HashKey* ComputeHash(const Val* index) const
-		{ return table_hash->ComputeHash(index, 1); }
 
 	bool AddProperties(Properties arg_state);
 	bool RemoveProperties(Properties arg_state);

src/const.bif

@@ -12,3 +12,4 @@ const NFS3::return_data: bool;
 const NFS3::return_data_max: count;
 const NFS3::return_data_first_only: bool;
 
+const Threading::heart_beat_interval: interval;

src/input.bif

@@ -0,0 +1,55 @@
+# functions and types for the input framework
+
+module Input;
+
+%%{
+#include "input/Manager.h"
+#include "NetVar.h"
+%%}
+
+type TableDescription: record;
+type EventDescription: record;
+
+function Input::__create_table_stream%(description: Input::TableDescription%) : bool
+	%{
+	bool res = input_mgr->CreateTableStream(description->AsRecordVal());
+	return new Val( res, TYPE_BOOL );
+	%}
+
+function Input::__create_event_stream%(description: Input::EventDescription%) : bool
+	%{
+	bool res = input_mgr->CreateEventStream(description->AsRecordVal());
+	return new Val( res, TYPE_BOOL );
+	%}
+
+function Input::__remove_stream%(id: string%) : bool
+	%{
+	bool res = input_mgr->RemoveStream(id->AsString()->CheckString());
+	return new Val( res, TYPE_BOOL );
+	%}
+
+function Input::__force_update%(id: string%) : bool
+	%{
+	bool res = input_mgr->ForceUpdate(id->AsString()->CheckString());
+	return new Val( res, TYPE_BOOL );
+	%}
+
+# Options for Ascii Reader
+
+module InputAscii;
+
+const separator: string;
+const set_separator: string;
+const empty_field: string;
+const unset_field: string;
+
+module InputRaw;
+const record_separator: string;
+
+module InputBenchmark;
+const factor: double;
+const spread: count;
+const autospread: double;
+const addfactor: count;
+const stopspreadat: count;
+const timedspread: double;

src/input/Manager.h

@@ -0,0 +1,180 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Class for managing input streams and filters
+
+#ifndef INPUT_MANAGER_H
+#define INPUT_MANAGER_H
+
+#include "../BroString.h"
+
+#include "../Val.h"
+#include "../EventHandler.h"
+#include "../RemoteSerializer.h"
+
+#include <map>
+
+namespace input {
+
+class ReaderFrontend;
+class ReaderBackend; 
+
+/**
+ * Singleton class for managing input streams.
+ */
+class Manager {
+public:
+	/**
+	 * Constructor.
+	 */	
+	Manager();
+
+	/**
+	 * Destructor.
+	 */
+	~Manager();
+    
+	/**
+	 * Creates a new input stream.
+	 * Add a filter to an input source, which will write the data from the data source into 
+	 * a Bro table.
+	 * Add a filter to an input source, which sends events for read input data.
+	 *
+	 * @param id  The enum value corresponding the input stream.
+	 *
+	 * @param description A record of script type \c Input:StreamDescription.
+	 *
+	 * This method corresponds directly to the internal BiF defined in
+	 * input.bif, which just forwards here.
+	 */	
+    	bool CreateTableStream(RecordVal* description);
+    	bool CreateEventStream(RecordVal* description);
+
+
+	/**
+	 * Force update on a input stream.
+	 * Forces a re-read of the whole input source.
+	 * Usually used, when an input stream is opened in managed mode.
+	 * Otherwise, this can be used to trigger a input source check before a heartbeat message arrives.
+	 * May be ignored by the reader.
+	 *
+	 * @param id  The enum value corresponding the input stream.
+	 *
+	 * This method corresponds directly to the internal BiF defined in
+	 * input.bif, which just forwards here.
+	 */
+	bool ForceUpdate(const string &id);
+	
+	/**
+	 * Deletes an existing input stream
+	 *
+	 * @param id  The enum value corresponding the input stream.
+	 *
+	 * This method corresponds directly to the internal BiF defined in
+	 * input.bif, which just forwards here.
+	 */
+	bool RemoveStream(const string &id);	
+
+protected:
+	friend class ReaderFrontend;
+	friend class PutMessage;
+	friend class DeleteMessage;
+	friend class ClearMessage;
+	friend class SendEventMessage;
+	friend class SendEntryMessage;
+	friend class EndCurrentSendMessage;
+	friend class ReaderClosedMessage;
+
+	// For readers to write to input stream in direct mode (reporting new/deleted values directly)
+	// Functions take ownership of threading::Value fields
+	void Put(ReaderFrontend* reader, threading::Value* *vals);
+	void Clear(ReaderFrontend* reader);
+	bool Delete(ReaderFrontend* reader, threading::Value* *vals);
+
+	// for readers to write to input stream in indirect mode (manager is monitoring new/deleted values)
+	// Functions take ownership of threading::Value fields
+	void SendEntry(ReaderFrontend* reader, threading::Value* *vals);
+	void EndCurrentSend(ReaderFrontend* reader);
+	
+	// Allows readers to directly send Bro events.
+	// The num_vals and vals must be the same the named event expects.
+	// Takes ownership of threading::Value fields
+	bool SendEvent(const string& name, const int num_vals, threading::Value* *vals);
+
+	// Instantiates a new ReaderBackend of the given type (note that
+	// doing so creates a new thread!).	
+	ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type);	
+	
+	// Functions are called from the ReaderBackend to notify the manager, that a filter has been removed
+	// or a stream has been closed.
+	// Used to prevent race conditions where data for a specific filter is still in the queue when the 
+	// RemoveStream directive is executed by the main thread.
+	// This makes sure all data that has ben queued for a filter is still received.
+	bool RemoveStreamContinuation(ReaderFrontend* reader);
+	
+private:
+	class Stream;
+	class TableStream;
+	class EventStream;
+	
+    	bool CreateStream(Stream*, RecordVal* description);
+
+	// SendEntry implementation for Tablefilter
+	int SendEntryTable(Stream* i, const threading::Value* const *vals);	
+
+	// Put implementation for Tablefilter
+	int PutTable(Stream* i, const threading::Value* const *vals);	
+
+	// SendEntry and Put implementation for Eventfilter
+	int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals);
+
+	// Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types 
+	// from the log framework
+	bool IsCompatibleType(BroType* t, bool atomic_only=false);
+
+	// Check if a record is made up of compatible types and return a list of all fields that are in the record in order.
+	// Recursively unrolls records
+	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend);
+
+	// Send events
+	void SendEvent(EventHandlerPtr ev, const int numvals, ...);	
+	void SendEvent(EventHandlerPtr ev, list<Val*> events);	
+
+	// Call predicate function and return result
+	bool CallPred(Func* pred_func, const int numvals, ...);
+
+	// get a hashkey for a set of threading::Values
+	HashKey* HashValues(const int num_elements, const threading::Value* const *vals);
+
+	// Get the memory used by a specific value
+	int GetValueLength(const threading::Value* val);
+	// Copies the raw data in a specific threading::Value to position sta
+	int CopyValue(char *data, const int startpos, const threading::Value* val);
+
+	// Convert Threading::Value to an internal Bro Type (works also with Records)
+	Val* ValueToVal(const threading::Value* val, BroType* request_type);
+
+	// Convert Threading::Value to an internal Bro List type
+	Val* ValueToIndexVal(int num_fields, const RecordType* type, const threading::Value* const *vals);
+
+	// Converts a threading::value to a record type. mostly used by ValueToVal
+	RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position);	
+	Val* RecordValToIndexVal(RecordVal *r);
+	
+	// Converts a Bro ListVal to a RecordVal given the record type
+	RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position);
+
+	Stream* FindStream(const string &name);
+	Stream* FindStream(ReaderFrontend* reader);
+
+	enum StreamType { TABLE_FILTER, EVENT_FILTER };
+	
+	map<ReaderFrontend*, Stream*> readers;
+};
+
+
+}
+
+extern input::Manager* input_mgr;
+
+
+#endif /* INPUT_MANAGER_H */

src/input/ReaderBackend.cc

@@ -0,0 +1,294 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "ReaderBackend.h"
+#include "ReaderFrontend.h"
+#include "Manager.h"
+
+using threading::Value;
+using threading::Field;
+
+namespace input {
+
+class PutMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	PutMessage(ReaderFrontend* reader, Value* *val)
+		: threading::OutputMessage<ReaderFrontend>("Put", reader),
+		val(val) {}
+
+	virtual bool Process() 
+		{
+		input_mgr->Put(Object(), val);
+		return true;
+		}
+
+private:
+	Value* *val;
+};
+
+class DeleteMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	DeleteMessage(ReaderFrontend* reader, Value* *val)
+		: threading::OutputMessage<ReaderFrontend>("Delete", reader),
+		val(val) {}
+
+	virtual bool Process() 
+		{
+		return input_mgr->Delete(Object(), val);
+		}
+
+private:
+	Value* *val;
+};
+
+class ClearMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	ClearMessage(ReaderFrontend* reader)
+		: threading::OutputMessage<ReaderFrontend>("Clear", reader) {}
+
+	virtual bool Process() 
+		{
+		input_mgr->Clear(Object());
+		return true;
+		}
+
+private:
+};
+
+class SendEventMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val)
+		: threading::OutputMessage<ReaderFrontend>("SendEvent", reader),
+		name(name), num_vals(num_vals), val(val) {}
+
+	virtual bool Process() 
+		{
+		bool success = input_mgr->SendEvent(name, num_vals, val);
+
+		if ( !success ) 
+			reporter->Error("SendEvent for event %s failed", name.c_str());
+
+		return true; // we do not want to die if sendEvent fails because the event did not return.
+		}
+
+private:
+	const string name;
+	const int num_vals;
+	Value* *val;
+};
+
+class SendEntryMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	SendEntryMessage(ReaderFrontend* reader, Value* *val)
+		: threading::OutputMessage<ReaderFrontend>("SendEntry", reader),
+		val(val) { }
+
+	virtual bool Process() 
+		{
+		input_mgr->SendEntry(Object(), val);
+		return true;
+		}
+
+private:
+	Value* *val;
+};
+
+class EndCurrentSendMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	EndCurrentSendMessage(ReaderFrontend* reader)
+		: threading::OutputMessage<ReaderFrontend>("EndCurrentSend", reader) {}
+
+	virtual bool Process() 
+		{
+		input_mgr->EndCurrentSend(Object());
+		return true;
+		}
+
+private:
+};
+
+class ReaderClosedMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	ReaderClosedMessage(ReaderFrontend* reader)
+		: threading::OutputMessage<ReaderFrontend>("ReaderClosed", reader) {}
+
+	virtual bool Process() 
+		{
+		return input_mgr->RemoveStreamContinuation(Object());
+		}
+
+private:
+};
+
+
+class DisableMessage : public threading::OutputMessage<ReaderFrontend>
+{
+public:
+        DisableMessage(ReaderFrontend* writer)
+		: threading::OutputMessage<ReaderFrontend>("Disable", writer)	{}
+
+	virtual bool Process()
+		{ 
+		Object()->SetDisable(); 
+		return true; 
+		}
+};
+
+
+ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread()
+	{
+	buf = 0;
+	buf_len = 1024;
+	disabled = true; // disabled will be set correcty in init.
+
+	frontend = arg_frontend;
+
+	SetName(frontend->Name());
+	}
+
+ReaderBackend::~ReaderBackend() 
+	{	
+	}
+
+void ReaderBackend::Put(Value* *val) 
+	{
+	SendOut(new PutMessage(frontend, val));
+	}
+
+void ReaderBackend::Delete(Value* *val) 
+	{
+	SendOut(new DeleteMessage(frontend, val));
+	}
+
+void ReaderBackend::Clear() 
+	{
+	SendOut(new ClearMessage(frontend));
+	}
+
+void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) 
+	{
+	SendOut(new SendEventMessage(frontend, name, num_vals, vals));
+	} 
+
+void ReaderBackend::EndCurrentSend() 
+	{
+	SendOut(new EndCurrentSendMessage(frontend));
+	}
+
+void ReaderBackend::SendEntry(Value* *vals)
+	{
+	SendOut(new SendEntryMessage(frontend, vals));
+	}
+
+bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, 
+		         const threading::Field* const* arg_fields) 
+	{
+	source = arg_source;
+	SetName("InputReader/"+source);
+
+	num_fields = arg_num_fields;
+	fields = arg_fields;	
+
+	// disable if DoInit returns error.
+	int success = DoInit(arg_source, mode, arg_num_fields, arg_fields);
+
+	if ( !success ) 
+		{
+		Error("Init failed");
+		DisableFrontend();
+		}
+
+	disabled = !success;
+
+	return success;
+	}
+
+void ReaderBackend::Close() 
+	{
+	DoClose();
+	disabled = true;
+	DisableFrontend();
+	SendOut(new ReaderClosedMessage(frontend));
+
+	if ( fields != 0 ) 
+		{
+		for ( unsigned int i = 0; i < num_fields; i++ ) 
+			delete(fields[i]);
+
+		delete[] (fields);
+		fields = 0;
+		}
+	}
+
+bool ReaderBackend::Update() 
+	{
+	if ( disabled ) 
+		return false;
+
+	bool success = DoUpdate();
+	if ( !success ) 
+		DisableFrontend();
+
+	return success;
+	}
+
+void ReaderBackend::DisableFrontend()
+	{
+	disabled = true; 
+	// we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now
+	SendOut(new DisableMessage(frontend));
+	}
+
+bool ReaderBackend::DoHeartbeat(double network_time, double current_time)
+	{
+	MsgThread::DoHeartbeat(network_time, current_time);
+	return true;
+	}
+
+TransportProto ReaderBackend::StringToProto(const string &proto) 
+	{
+	if ( proto == "unknown" ) 
+		return TRANSPORT_UNKNOWN;
+	else if ( proto == "tcp" )
+		return TRANSPORT_TCP;
+	else if ( proto == "udp" )
+		return TRANSPORT_UDP;
+	else if ( proto == "icmp" )
+		return TRANSPORT_ICMP;
+
+	Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
+
+	return TRANSPORT_UNKNOWN;
+	}
+
+
+// more or less verbose copy from IPAddr.cc -- which uses reporter
+Value::addr_t ReaderBackend::StringToAddr(const string &s) 
+	{
+	Value::addr_t val;
+
+	if ( s.find(':') == std::string::npos ) // IPv4.
+		{
+		val.family = IPv4;
+
+		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) 
+			{
+			Error(Fmt("Bad addres: %s", s.c_str()));
+			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
+			}
+
+
+		}
+	else
+		{
+		val.family = IPv6;
+		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
+			{
+			Error(Fmt("Bad IP address: %s", s.c_str()));
+			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
+			}
+		}
+
+	return val;
+	}
+
+}

src/input/ReaderBackend.h

@@ -0,0 +1,237 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef INPUT_READERBACKEND_H
+#define INPUT_READERBACKEND_H
+
+#include "BroString.h"
+#include "../threading/SerialTypes.h"
+#include "threading/MsgThread.h"
+
+namespace input {
+
+class ReaderFrontend;
+
+/**
+ * Base class for reader implementation. When the input:Manager creates a
+ * new input stream, it instantiates a ReaderFrontend. That then in turn
+ * creates a ReaderBackend of the right type. The frontend then forwards
+ * message over the backend as its methods are called.
+ *
+ * All of this methods must be called only from the corresponding child
+ * thread (the constructor is the one exception.)
+ */
+class ReaderBackend : public threading::MsgThread {
+public:
+	/**
+	 * Constructor.
+	 *
+	 * @param frontend The frontend reader that created this backend. The
+	 * *only* purpose of this value is to be passed back via messages as
+	 * a argument to callbacks. One must not otherwise access the
+	 * frontend, it's running in a different thread.
+	 *
+	 * @param frontend pointer to the reader frontend
+	 */	
+	ReaderBackend(ReaderFrontend* frontend);
+    	
+	/**
+	 * Destructor.
+	 */	
+	virtual ~ReaderBackend();
+
+	/**
+	 * One-time initialization of the reader to define the input source.
+	 *
+	 * @param arg_source A string left to the interpretation of the reader
+	 * implementation; it corresponds to the value configured on the
+	 * script-level for the input stream.
+	 *
+	 * @param num_fields The number of log fields for the stream.
+	 *
+	 * @param fields An array of size \a num_fields with the log fields.
+	 * The methods takes ownership of the array.
+	 * 
+	 * @param mode the opening mode for the input source
+	 *
+	 * @param arg_num_fields number of fields contained in \a fields
+	 *
+	 * @param fields the types and names of the fields to be retrieved from the input source
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields);
+
+	/**
+	 * Finishes reading from this input stream in a regular fashion. Must not be
+	 * called if an error has been indicated earlier. After calling this,
+	 * no further reading from the stream can be performed
+	 *
+	 * @return False if an error occured.
+	 */
+	void Close();
+
+	/**
+	 * Force trigger an update of the input stream.
+	 * The action that will be taken depends on the current read mode and the individual input backend
+	 *
+	 * An backend can choose to ignore this.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Update();
+
+	/**
+	 * Disables the frontend that has instantiated this backend. Once
+	 * disabled,the frontend will not send any further message over.
+	 */
+	void DisableFrontend();	
+	
+protected:
+    	// Methods that have to be overwritten by the individual readers
+	
+	/**
+	 * Reader-specific intialization method. Note that data may only be read from the input source
+	 * after the Start function has been called.
+	 *
+	 * A reader implementation must override this method. If it returns
+	 * false, it will be assumed that a fatal error has occured that
+	 * prevents the reader from further operation; it will then be
+	 * disabled and eventually deleted. When returning false, an
+	 * implementation should also call Error() to indicate what happened.
+	 */
+	virtual bool DoInit(string arg_sources, int mode, int arg_num_fields, const threading::Field* const* fields) = 0;
+
+	/**
+	 * Reader-specific method implementing input finalization at
+	 * termination. 
+	 *
+	 * A reader implementation must override this method but it can just
+	 * ignore calls if an input source  must not be closed.
+	 *
+	 * After the method is called, the writer will be deleted. If an error occurs
+	 * during shutdown, an implementation should also call Error() to indicate what
+	 * happened.
+	 */	
+	virtual void DoClose() = 0;
+
+	/**
+	 * Reader-specific method implementing the forced update trigger
+	 *
+	 * A reader implementation must override this method but it can just ignore
+	 * calls, if a forced update does not fit the input source or the current input
+	 * reading mode.
+	 *
+	 * If it returns false, it will be assumed that a fatal error has occured
+	 * that prevents the reader from further operation; it will then be
+	 * disabled and eventually deleted. When returning false, an implementation
+	 * should also call Error to indicate what happened.
+	 */
+	virtual bool DoUpdate() = 0;
+	
+	/**
+	 * Returns the input source as passed into the constructor.
+	 */
+	const string Source() const	{ return source; }
+
+	/**
+	 * Method allowing a reader to send a specified bro event.
+	 * Vals must match the values expected by the bro event.
+	 *
+	 * @param name name of the bro event to send
+	 *
+	 * @param num_vals number of entries in \a vals
+	 *
+	 * @param vals the values to be given to the event
+	 */
+	void SendEvent(const string& name, const int num_vals, threading::Value* *vals);
+
+	// Content-sending-functions (simple mode). Including table-specific stuff that simply is not used if we have no table
+	/**
+	 * Method allowing a reader to send a list of values read for a specific filter back to the manager.
+	 *
+	 * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised
+	 *
+	 * @param val list of threading::Values expected by the filter
+	 */
+	void Put(threading::Value* *val);
+
+	/**
+	 * Method allowing a reader to delete a specific value from a bro table.
+	 *
+	 * If the receiving filter is an event, only a removed event is raised
+	 *
+	 * @param val list of threading::Values expected by the filter
+	 */
+	void Delete(threading::Value* *val);
+
+	/**
+	 * Method allowing a reader to clear a value from a bro table.
+	 *
+	 * If the receiving filter is an event, this is ignored.
+	 *
+	 */
+	void Clear();
+
+	// Content-sending-functions (tracking mode): Only changed lines are propagated.
+	
+
+	/**
+	 * Method allowing a reader to send a list of values read for a specific filter back to the manager.
+	 *
+	 * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised.
+	 *
+	 * @param val list of threading::Values expected by the filter
+	 */
+	void SendEntry(threading::Value*  *vals);
+
+	/**
+	 * Method telling the manager, that the current list of entries sent by SendEntry is finished.
+	 * 
+	 * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer
+	 * present in the input source
+	 *
+	 */
+	void EndCurrentSend();
+
+	/**
+	 * Triggered by regular heartbeat messages from the main thread.
+	 *
+	 * This method can be overridden but once must call
+	 * ReaderBackend::DoHeartbeat().
+	 */
+	virtual bool DoHeartbeat(double network_time, double current_time);	
+
+	/**
+	 * Utility function for Readers - convert a string into a TransportProto
+	 *
+	 * @param proto the transport protocol
+	 */
+	TransportProto StringToProto(const string &proto);		
+
+	/**
+	 * Utility function for Readers - convert a string into a Value::addr_t
+	 *
+	 * @param addr containing an ipv4 or ipv6 address
+	 */
+	threading::Value::addr_t StringToAddr(const string &addr);
+
+private:
+	// Frontend that instantiated us. This object must not be access from
+	// this class, it's running in a different thread!
+	ReaderFrontend* frontend;	
+
+	string source;
+    
+    	bool disabled;
+
+	// For implementing Fmt().
+	char* buf;
+	unsigned int buf_len;
+
+	unsigned int num_fields;
+	const threading::Field* const * fields; // raw mapping		
+};
+
+}
+
+#endif /* INPUT_READERBACKEND_H */

src/input/ReaderFrontend.cc

@@ -0,0 +1,115 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Manager.h"
+#include "ReaderFrontend.h"
+#include "ReaderBackend.h"
+#include "threading/MsgThread.h"
+
+// FIXME: cleanup of disabled inputreaders is missing. we need this, because stuff can e.g. fail in init and might never be removed afterwards.
+
+namespace input {
+
+class InitMessage : public threading::InputMessage<ReaderBackend>
+{
+public:
+	InitMessage(ReaderBackend* backend, const string source, const int mode, const int num_fields, const threading::Field* const* fields)
+		: threading::InputMessage<ReaderBackend>("Init", backend),
+		source(source), mode(mode), num_fields(num_fields), fields(fields) { }
+
+	virtual bool Process() { return Object()->Init(source, mode, num_fields, fields); }
+
+private:
+	const string source;
+	const int mode;
+	const int num_fields;
+       	const threading::Field* const* fields;
+};
+
+class UpdateMessage : public threading::InputMessage<ReaderBackend>
+{
+public:
+	UpdateMessage(ReaderBackend* backend)
+		: threading::InputMessage<ReaderBackend>("Update", backend)
+		 { }
+
+	virtual bool Process() { return Object()->Update(); }
+};
+
+class CloseMessage : public threading::InputMessage<ReaderBackend>
+{
+public:
+	CloseMessage(ReaderBackend* backend)
+		: threading::InputMessage<ReaderBackend>("Close", backend)
+		 { }
+
+	virtual bool Process() { Object()->Close(); return true; }
+};
+
+
+ReaderFrontend::ReaderFrontend(bro_int_t type) 
+	{
+	disabled = initialized = false;
+	ty_name = "<not set>";
+	backend = input_mgr->CreateBackend(this, type);
+
+	assert(backend);
+	backend->Start();
+	}
+
+ReaderFrontend::~ReaderFrontend() 
+	{
+	}
+
+void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, 
+		          const threading::Field* const* fields) 
+	{
+	if ( disabled )
+		return;
+
+	if ( initialized )
+		reporter->InternalError("reader initialize twice");
+
+	source = arg_source;
+	initialized = true;
+
+	backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields));
+	} 
+
+void ReaderFrontend::Update() 
+	{
+	if ( disabled ) 
+		return;
+
+	if ( !initialized ) 
+		{
+		reporter->Error("Tried to call update on uninitialized reader");
+		return;
+		}
+
+	backend->SendIn(new UpdateMessage(backend));
+	}
+
+void ReaderFrontend::Close() 
+	{
+	if ( disabled ) 
+		return;
+	
+	if ( !initialized ) 
+		{
+		reporter->Error("Tried to call finish on uninitialized reader");
+		return;
+		}
+
+	backend->SendIn(new CloseMessage(backend));
+	}
+
+string ReaderFrontend::Name() const
+	{
+	if ( source.size() )
+		return ty_name;
+
+	return ty_name + "/" + source;
+	}
+
+}
+

src/input/ReaderFrontend.h

@@ -0,0 +1,123 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef INPUT_READERFRONTEND_H
+#define INPUT_READERFRONTEND_H
+
+#include "../threading/MsgThread.h"
+#include "../threading/SerialTypes.h"
+
+namespace input {
+
+class Manager;
+
+/**
+ * Bridge class between the input::Manager and backend input threads. The
+ * Manager instantiates one \a ReaderFrontend for each open input stream.
+ * Each frontend in turns instantiates a ReaderBackend-derived class
+ * internally that's specific to the particular input format. That backend
+ * spawns a new thread, and it receives messages from the frontend that
+ * correspond to method called by the manager.
+ */
+class ReaderFrontend {
+public:
+	/**
+	 * Constructor.
+	 *
+	 * type: The backend writer type, with the value corresponding to the
+	 * script-level \c Input::Reader enum (e.g., \a READER_ASCII). The
+	 * frontend will internally instantiate a ReaderBackend of the
+	 * corresponding type.
+	 *
+	 * Frontends must only be instantiated by the main thread.
+	 */	
+	ReaderFrontend(bro_int_t type);
+
+	/**
+	 * Destructor.
+	 *
+	 * Frontends must only be destroyed by the main thread.
+	 */	
+	virtual ~ReaderFrontend();
+
+	/**
+	 * Initializes the reader.
+	 *
+	 * This method generates a message to the backend reader and triggers
+	 * the corresponding message there. If the backend method fails, it
+	 * sends a message back that will asynchronously call Disable().
+	 *
+	 * See ReaderBackend::Init() for arguments.
+	 * This method must only be called from the main thread.
+	 */	
+	void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields);
+
+	/**
+	 * Force an update of the current input source. Actual action depends on
+	 * the opening mode and on the input source.
+	 *
+	 * This method generates a message to the backend reader and triggers
+	 * the corresponding message there.
+	 * This method must only be called from the main thread.
+	 */
+	void Update();
+
+	/**
+	 * Finalizes writing to this tream.
+	 *
+	 * This method generates a message to the backend reader and triggers
+	 * the corresponding message there.
+	 * This method must only be called from the main thread.
+	 */	
+	void Close();
+
+	/**
+	 * Disables the reader frontend. From now on, all method calls that
+	 * would normally send message over to the backend, turn into no-ops.
+	 * Note though that it does not stop the backend itself, use Finsh()
+	 * to do that as well (this method is primarily for use as callback
+	 * when the backend wants to disable the frontend).
+	 *
+	 * Disabled frontend will eventually be discarded by the
+	 * input::Manager.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void SetDisable()	{ disabled = true; }
+
+	/**
+	 * Returns true if the reader frontend has been disabled with SetDisable().
+	 */
+	bool Disabled()	{ return disabled; }	
+
+	/**
+	 * Returns a descriptive name for the reader, including the type of
+	 * the backend and the source used.
+	 *
+	 * This method is safe to call from any thread.
+	 */
+	string Name() const;
+
+protected:
+	friend class Manager;
+
+	/**
+	 * Returns the source as passed into the constructor
+	 */
+	const string Source() const	{ return source; };	
+
+	string ty_name;	// Name of the backend type. Set by the manager.
+
+private:
+	ReaderBackend* backend;	// The backend we have instanatiated.	
+	string source;
+	bool disabled;		// True if disabled.
+	bool initialized;	// True if initialized.	
+
+};
+
+}
+
+
+#endif /* INPUT_READERFRONTEND_H */
+
+

src/input/fdstream.h

@@ -0,0 +1,189 @@
+/* The following code declares classes to read from and write to
+ * file descriptore or file handles.
+ *
+ * See
+ *      http://www.josuttis.com/cppcode
+ * for details and the latest version.
+ *
+ * - open:
+ *      - integrating BUFSIZ on some systems?
+ *      - optimized reading of multiple characters
+ *      - stream for reading AND writing
+ *      - i18n
+ *
+ * (C) Copyright Nicolai M. Josuttis 2001.
+ * Permission to copy, use, modify, sell and distribute this software
+ * is granted provided this copyright notice appears in all copies.
+ * This software is provided "as is" without express or implied
+ * warranty, and with no claim as to its suitability for any purpose.
+ *
+ * Version: Jul 28, 2002
+ * History:
+ *  Jul 28, 2002: bugfix memcpy() => memmove()
+ *                fdinbuf::underflow(): cast for return statements
+ *  Aug 05, 2001: first public version
+ */
+#ifndef BOOST_FDSTREAM_HPP
+#define BOOST_FDSTREAM_HPP
+
+#include <istream>
+#include <ostream>
+#include <streambuf>
+// for EOF:
+#include <cstdio>
+// for memmove():
+#include <cstring>
+
+
+
+// low-level read and write functions
+#ifdef _MSC_VER
+# include <io.h>
+#else
+# include <sys/errno.h>
+# include <unistd.h>
+//extern "C" {
+//    int write (int fd, const char* buf, int num);
+//    int read (int fd, char* buf, int num);
+//}
+#endif
+
+
+// BEGIN namespace BOOST
+namespace boost {
+
+
+/************************************************************
+ * fdostream
+ * - a stream that writes on a file descriptor
+ ************************************************************/
+
+
+class fdoutbuf : public std::streambuf {
+  protected:
+    int fd;    // file descriptor
+  public:
+    // constructor
+    fdoutbuf (int _fd) : fd(_fd) {
+    }
+  protected:
+    // write one character
+    virtual int_type overflow (int_type c) {
+        if (c != EOF) {
+            char z = c;
+            if (write (fd, &z, 1) != 1) {
+                return EOF;
+            }
+        }
+        return c;
+    }
+    // write multiple characters
+    virtual
+    std::streamsize xsputn (const char* s,
+                            std::streamsize num) {
+        return write(fd,s,num);
+    }
+};
+
+class fdostream : public std::ostream {
+  protected:
+    fdoutbuf buf;
+  public:
+    fdostream (int fd) : std::ostream(0), buf(fd) {
+        rdbuf(&buf);
+    }
+};
+
+
+/************************************************************
+ * fdistream
+ * - a stream that reads on a file descriptor
+ ************************************************************/
+
+class fdinbuf : public std::streambuf {
+  protected:
+    int fd;    // file descriptor
+  protected:
+    /* data buffer:
+     * - at most, pbSize characters in putback area plus
+     * - at most, bufSize characters in ordinary read buffer
+     */
+    static const int pbSize = 4;        // size of putback area
+    static const int bufSize = 1024;    // size of the data buffer
+    char buffer[bufSize+pbSize];        // data buffer
+
+  public:
+    /* constructor
+     * - initialize file descriptor
+     * - initialize empty data buffer
+     * - no putback area
+     * => force underflow()
+     */
+    fdinbuf (int _fd) : fd(_fd) {
+        setg (buffer+pbSize,     // beginning of putback area
+              buffer+pbSize,     // read position
+              buffer+pbSize);    // end position
+    }
+
+  protected:
+    // insert new characters into the buffer
+    virtual int_type underflow () {
+#ifndef _MSC_VER
+        using std::memmove;
+#endif
+
+        // is read position before end of buffer?
+        if (gptr() < egptr()) {
+            return traits_type::to_int_type(*gptr());
+        }
+
+        /* process size of putback area
+         * - use number of characters read
+         * - but at most size of putback area
+         */
+        int numPutback;
+        numPutback = gptr() - eback();
+        if (numPutback > pbSize) {
+            numPutback = pbSize;
+        }
+
+        /* copy up to pbSize characters previously read into
+         * the putback area
+         */
+        memmove (buffer+(pbSize-numPutback), gptr()-numPutback,
+                numPutback);
+
+        // read at most bufSize new characters
+        int num;
+        num = read (fd, buffer+pbSize, bufSize);
+	if ( num == EAGAIN ) {
+		return 0;
+	}
+        if (num <= 0) {
+            // ERROR or EOF
+            return EOF;
+        }
+
+        // reset buffer pointers
+        setg (buffer+(pbSize-numPutback),   // beginning of putback area
+              buffer+pbSize,                // read position
+              buffer+pbSize+num);           // end of buffer
+
+        // return next character
+        return traits_type::to_int_type(*gptr());
+    }
+};
+
+class fdistream : public std::istream {
+  protected:
+    fdinbuf buf;
+  public:
+    fdistream (int fd) : std::istream(0), buf(fd) {
+        rdbuf(&buf);
+    }
+};
+
+
+} // END namespace boost
+
+#endif /*BOOST_FDSTREAM_HPP*/

src/input/readers/Ascii.cc

@@ -0,0 +1,527 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Ascii.h"
+#include "NetVar.h"
+
+#include <fstream>
+#include <sstream>
+
+#include "../../threading/SerialTypes.h"
+
+#define MANUAL 0
+#define REREAD 1
+#define STREAM 2
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+using namespace input::reader;
+using threading::Value;
+using threading::Field;
+
+
+FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) 
+	: name(arg_name), type(arg_type)
+	{
+	position = arg_position;
+	secondary_position = -1;
+	present = true;
+	}
+
+FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, 
+		const TypeTag& arg_subtype, int arg_position) 
+	: name(arg_name), type(arg_type), subtype(arg_subtype)
+	{
+	position = arg_position;
+	secondary_position = -1;
+	present = true;
+	}
+
+FieldMapping::FieldMapping(const FieldMapping& arg) 
+	: name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present)
+	{
+	position = arg.position;
+	secondary_position = arg.secondary_position;
+	}
+
+FieldMapping FieldMapping::subType() 
+	{
+	return FieldMapping(name, subtype, position);
+	}
+
+Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
+	{
+	file = 0;
+
+	
+	separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), 
+			  BifConst::InputAscii::separator->Len());
+	if ( separator.size() != 1 ) 
+		Error("separator length has to be 1. Separator will be truncated.");
+
+	set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(),
+		              BifConst::InputAscii::set_separator->Len());
+	if ( set_separator.size() != 1 ) 
+		Error("set_separator length has to be 1. Separator will be truncated.");
+
+	empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), 
+			    BifConst::InputAscii::empty_field->Len());
+	
+	unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), 
+			    BifConst::InputAscii::unset_field->Len());
+
+}
+
+Ascii::~Ascii()
+	{
+	DoClose();
+	}
+
+void Ascii::DoClose()
+	{
+	if ( file != 0 ) 
+		{
+		file->close();
+		delete(file);
+		file = 0;
+		}
+	}
+
+bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields)
+	{
+	fname = path;
+	mode = arg_mode;
+	mtime = 0;
+	
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) 
+		{
+		Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str()));
+		return false;
+		} 
+
+	file = new ifstream(path.c_str());
+	if ( !file->is_open() ) 
+		{
+		Error(Fmt("Init: cannot open %s", fname.c_str()));
+		delete(file);
+		file = 0;
+		return false;
+		}
+	
+	if ( ReadHeader(false) == false ) 
+		{
+		Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str()));
+		file->close();
+		delete(file);
+		file = 0;
+		return false;
+		}
+	
+	DoUpdate();
+
+	return true;
+	}
+
+
+bool Ascii::ReadHeader(bool useCached) 
+	{
+	// try to read the header line...
+	string line;
+	map<string, uint32_t> ifields;
+
+	if ( !useCached ) 
+		{
+		if ( !GetLine(line) ) 
+			{
+			Error("could not read first line");
+			return false;
+			}
+
+		headerline = line;
+		}
+	else 
+		line = headerline;
+	
+	// construct list of field names.
+	istringstream splitstream(line);
+	int pos=0;
+	while ( splitstream ) 
+		{
+		string s;
+		if ( !getline(splitstream, s, separator[0]))
+			break;
+
+		ifields[s] = pos;
+		pos++;
+		}
+
+	//printf("Updating fields from description %s\n", line.c_str());
+	columnMap.clear();
+		
+	for ( unsigned int i = 0; i < num_fields; i++ ) 
+		{
+		const Field* field = fields[i];
+		
+		map<string, uint32_t>::iterator fit = ifields.find(field->name);	
+		if ( fit == ifields.end() ) 
+			{
+			if ( field->optional ) 
+				{
+				// we do not really need this field. mark it as not present and always send an undef back.
+				FieldMapping f(field->name, field->type, field->subtype, -1);
+				f.present = false;
+				columnMap.push_back(f);
+				continue;
+				}
+
+			Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str()));
+			return false;
+			}
+
+
+		FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]);
+		if ( field->secondary_name != "" ) 
+			{
+			map<string, uint32_t>::iterator fit2 = ifields.find(field->secondary_name);					
+			if ( fit2 == ifields.end() ) 
+				{
+				Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str()));
+				return false;
+				}
+			f.secondary_position = ifields[field->secondary_name];
+			}
+		columnMap.push_back(f);
+		}
+
+	
+	// well, that seems to have worked...
+	return true;
+	}
+
+bool Ascii::GetLine(string& str) 
+	{
+	while ( getline(*file, str) )
+       		{
+		if ( str[0] != '#' ) 
+			return true;
+
+		if ( str.compare(0,8, "#fields\t") == 0 ) 
+			{
+			str = str.substr(8);
+			return true;
+			}
+		}
+
+	return false;
+	}
+
+
+Value* Ascii::EntryToVal(string s, FieldMapping field) 
+	{
+
+	if ( s.compare(unset_field) == 0 )  // field is not set...
+		return new Value(field.type, false);
+
+	Value* val = new Value(field.type, true);
+	
+	switch ( field.type ) {
+	case TYPE_ENUM:
+	case TYPE_STRING:
+		val->val.string_val = new string(s);
+		break;
+
+	case TYPE_BOOL:
+		if ( s == "T" ) 
+			val->val.int_val = 1;
+		else if ( s == "F" ) 
+			val->val.int_val = 0;
+		else 
+			{
+			Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str()));
+			return false;
+			}
+		break;
+
+	case TYPE_INT:
+		val->val.int_val = atoi(s.c_str());
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		val->val.double_val = atof(s.c_str());
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = atoi(s.c_str());
+		break;
+
+	case TYPE_PORT:
+		val->val.port_val.port = atoi(s.c_str());
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		break;
+
+	case TYPE_SUBNET: 
+		{
+		size_t pos = s.find("/");
+		if ( pos == s.npos ) {
+			Error(Fmt("Invalid value for subnet: %s", s.c_str()));
+			return false;
+		}
+		int width = atoi(s.substr(pos+1).c_str());
+		string addr = s.substr(0, pos);
+
+		val->val.subnet_val.prefix = StringToAddr(addr);
+		val->val.subnet_val.length = width;
+		break;
+		}
+
+	case TYPE_ADDR: 
+		val->val.addr_val = StringToAddr(s);
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		// First - common initialization
+		// Then - initialization for table.
+		// Then - initialization for vector.
+		// Then - common stuff
+		{
+		// how many entries do we have...
+		unsigned int length = 1;
+		for ( unsigned int i = 0; i < s.size(); i++ )
+			if ( s[i] == ',') length++;
+
+		unsigned int pos = 0;
+		
+		if ( s.compare(empty_field) == 0 ) 
+			length = 0;
+
+		Value** lvals = new Value* [length];
+
+		if ( field.type == TYPE_TABLE ) 
+			{
+			val->val.set_val.vals = lvals;
+			val->val.set_val.size = length;
+			}
+		else if ( field.type == TYPE_VECTOR ) 
+			{
+			val->val.vector_val.vals = lvals;
+			val->val.vector_val.size = length;
+			} 
+		else 
+			{
+			assert(false);
+			}
+
+		if ( length == 0 )
+			break; //empty
+
+		istringstream splitstream(s);
+		while ( splitstream ) 
+			{
+			string element;
+
+			if ( !getline(splitstream, element, set_separator[0]) )
+				break;
+
+			if ( pos >= length ) 
+				{
+				Error(Fmt("Internal error while parsing set. pos %d >= length %d."
+				          " Element: %s", pos, length, element.c_str()));
+				break;
+				}
+
+			Value* newval = EntryToVal(element, field.subType());
+			if ( newval == 0 ) 
+				{
+				Error("Error while reading set");
+				return 0;
+				}
+			lvals[pos] = newval;
+
+			pos++;
+			}
+
+
+		if ( pos != length ) 
+			{
+			Error("Internal error while parsing set: did not find all elements");
+			return 0;
+			}
+
+		break;
+		}
+
+
+	default:
+		Error(Fmt("unsupported field format %d for %s", field.type,
+		field.name.c_str()));
+		return 0;
+	}	
+
+	return val;
+	}
+
+// read the entire file and send appropriate thingies back to InputMgr
+bool Ascii::DoUpdate() 
+	{
+	switch ( mode ) {
+		case REREAD:
+			// check if the file has changed
+			struct stat sb;
+			if ( stat(fname.c_str(), &sb) == -1 ) 
+				{
+				Error(Fmt("Could not get stat for %s", fname.c_str()));
+				return false;
+				}
+
+			if ( sb.st_mtime <= mtime ) // no change
+				return true;
+
+			mtime = sb.st_mtime;
+			// file changed. reread.
+
+			// fallthrough
+		case MANUAL:
+		case STREAM:
+
+			// dirty, fix me. (well, apparently after trying seeking, etc 
+			// - this is not that bad)
+			if ( file && file->is_open() ) 
+				{
+				if ( mode == STREAM ) 
+					{
+					file->clear(); // remove end of file evil bits
+					if ( !ReadHeader(true) )  // in case filters changed
+						return false; // header reading failed
+
+					break;
+					}
+				file->close();
+				}
+			file = new ifstream(fname.c_str());
+			if ( !file->is_open() ) 
+				{
+				Error(Fmt("cannot open %s", fname.c_str()));
+				return false;
+				}
+
+
+			if ( ReadHeader(false) == false ) 
+				{
+				return false;
+				}
+
+			break;
+		default:
+			assert(false);
+
+		}
+
+	string line;
+	while ( GetLine(line ) ) 
+		{
+		// split on tabs
+		istringstream splitstream(line);
+
+		map<int, string> stringfields;
+		int pos = 0;
+		while ( splitstream ) 
+			{
+			string s;
+			if ( !getline(splitstream, s, separator[0]) )
+				break;
+
+			stringfields[pos] = s;
+			pos++;
+			}
+
+		pos--; // for easy comparisons of max element.
+
+		
+		Value** fields = new Value*[num_fields];
+
+		int fpos = 0;
+		for ( vector<FieldMapping>::iterator fit = columnMap.begin();
+			fit != columnMap.end();
+			fit++ )
+			{
+
+			if ( ! fit->present ) 
+				{
+				// add non-present field
+				fields[fpos] =  new Value((*fit).type, false);
+				fpos++;
+				continue;
+				}
+			
+			assert(fit->position >= 0 );
+
+			if ( (*fit).position > pos || (*fit).secondary_position > pos ) 
+				{
+				Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos,  (*fit).position, (*fit).secondary_position));
+				return false;
+				}
+
+			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
+			if ( val == 0 ) 
+				{
+				Error("Could not convert String value to Val");
+				return false;
+				}
+			
+			if ( (*fit).secondary_position != -1 ) 
+				{
+				// we have a port definition :)
+				assert(val->type == TYPE_PORT ); 
+				//	Error(Fmt("Got type %d != PORT with secondary position!", val->type));
+
+				val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]);
+				}
+
+			fields[fpos] = val;
+
+			fpos++;
+			}
+
+		//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
+		assert ( (unsigned int) fpos == num_fields );
+
+		if ( mode == STREAM ) 
+			Put(fields);
+		else
+			SendEntry(fields);
+		}
+
+	if ( mode != STREAM ) 
+		EndCurrentSend();
+	
+	return true;
+	}
+
+bool Ascii::DoHeartbeat(double network_time, double current_time)
+{
+	ReaderBackend::DoHeartbeat(network_time, current_time);
+	
+	switch ( mode ) {
+		case MANUAL:
+			// yay, we do nothing :)
+			break;
+		case REREAD:
+		case STREAM:
+			Update(); // call update and not DoUpdate, because update 
+				  // checks disabled.
+			break;
+		default:
+			assert(false);
+	}
+
+	return true;
+	}
+

src/input/readers/Ascii.h

@@ -0,0 +1,90 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef INPUT_READERS_ASCII_H
+#define INPUT_READERS_ASCII_H
+
+#include <iostream>
+#include <vector>
+
+#include "../ReaderBackend.h"
+
+namespace input { namespace reader {
+
+// Description for input field mapping
+struct FieldMapping {
+	string name;
+	TypeTag type;
+	// internal type for sets and vectors
+	TypeTag subtype;
+	int position;
+	// for ports: pos of the second field
+	int secondary_position;
+	bool present;
+
+	FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); 
+	FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); 
+	FieldMapping(const FieldMapping& arg);
+	FieldMapping() { position = -1; secondary_position = -1; }
+
+	FieldMapping subType();
+	//bool IsEmpty() { return position == -1; }
+};
+
+
+class Ascii : public ReaderBackend {
+public:
+    Ascii(ReaderFrontend* frontend);
+    ~Ascii();
+    
+    static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); }
+    
+protected:
+	
+	virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields);
+
+	virtual void DoClose();
+
+	virtual bool DoUpdate();
+
+private:
+
+	virtual bool DoHeartbeat(double network_time, double current_time);
+
+	unsigned int num_fields;
+
+	const threading::Field* const * fields; // raw mapping		
+
+	// map columns in the file to columns to send back to the manager
+	vector<FieldMapping> columnMap;		
+
+	bool ReadHeader(bool useCached);
+	threading::Value* EntryToVal(string s, FieldMapping type);
+
+	bool GetLine(string& str);
+	
+	ifstream* file;
+	string fname;
+
+	// Options set from the script-level.
+	string separator;
+
+	string set_separator;
+
+	string empty_field;
+
+	string unset_field;
+	
+	// keep a copy of the headerline to determine field locations when filters change
+	string headerline;
+
+	int mode;
+
+	time_t mtime;
+
+};
+
+
+}
+}
+
+#endif /* INPUT_READERS_ASCII_H */

src/input/readers/Benchmark.cc

@@ -0,0 +1,278 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Benchmark.h"
+#include "NetVar.h"
+
+#include "../../threading/SerialTypes.h"
+
+#define MANUAL 0
+#define REREAD 1
+#define STREAM 2
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "../../threading/Manager.h"
+
+using namespace input::reader;
+using threading::Value;
+using threading::Field;
+
+
+
+Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend)
+	{
+	multiplication_factor = double(BifConst::InputBenchmark::factor);	
+	autospread = double(BifConst::InputBenchmark::autospread);	
+	spread = int(BifConst::InputBenchmark::spread);
+	add = int(BifConst::InputBenchmark::addfactor);
+	autospread_time = 0;
+	stopspreadat = int(BifConst::InputBenchmark::stopspreadat);
+	timedspread = double(BifConst::InputBenchmark::timedspread);
+	heart_beat_interval = double(BifConst::Threading::heart_beat_interval);
+
+	}
+
+Benchmark::~Benchmark()
+	{
+	DoClose();
+	}
+
+void Benchmark::DoClose()
+	{
+	}
+
+bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields)
+	{
+	mode = arg_mode;
+	
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+	num_lines = atoi(path.c_str());
+	
+	if ( autospread != 0.0 )
+		autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
+
+	if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) 
+		{
+		Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str()));
+		return false;
+		}
+
+	heartbeatstarttime = CurrTime();
+	DoUpdate();
+
+	return true;
+	}
+
+string Benchmark::RandomString(const int len) 
+	{
+	string s(len, ' ');
+
+	static const char values[] =
+	"0123456789!@#$%^&*()-_=+{}[]\\|"
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	"abcdefghijklmnopqrstuvwxyz";
+
+	for (int i = 0; i < len; ++i) 
+        	s[i] = values[rand() / (RAND_MAX / sizeof(values))];
+
+	return s;
+	}
+
+double Benchmark::CurrTime() 
+	{
+	struct timeval tv;
+	assert ( gettimeofday(&tv, 0) >= 0 );
+
+	return double(tv.tv_sec) + double(tv.tv_usec) / 1e6;
+	}
+
+
+// read the entire file and send appropriate thingies back to InputMgr
+bool Benchmark::DoUpdate() 
+	{
+        int linestosend = num_lines * heart_beat_interval;
+	for ( int i = 0; i < linestosend; i++ ) 
+		{
+		Value** field = new Value*[num_fields];
+		for  (unsigned int j = 0; j < num_fields; j++ ) 
+			field[j] = EntryToVal(fields[j]->type, fields[j]->subtype);
+
+		if ( mode == STREAM ) 
+			// do not do tracking, spread out elements over the second that we have...
+			Put(field);
+		else 
+			SendEntry(field);
+		
+		if ( stopspreadat == 0 || num_lines < stopspreadat ) 
+			{
+			if ( spread != 0 ) 
+				usleep(spread);
+
+			if ( autospread_time != 0 ) 
+				usleep( autospread_time );
+			}
+
+		if ( timedspread != 0.0 ) 
+			{
+			double diff;
+			do 
+				diff = CurrTime() - heartbeatstarttime;
+			while ( diff/heart_beat_interval < i/(linestosend 
+			        + (linestosend * timedspread) ) );
+			}
+
+	}
+
+	if ( mode != STREAM )  
+		EndCurrentSend();
+
+	return true;
+}
+
+threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) 
+	{
+	Value* val = new Value(type, true);
+
+	// basically construct something random from the fields that we want.
+	
+	switch ( type ) {
+	case TYPE_ENUM:
+		assert(false); // no enums, please.
+	case TYPE_STRING:
+		val->val.string_val = new string(RandomString(10));
+		break;
+
+	case TYPE_BOOL:
+		val->val.int_val = 1; // we never lie.
+		break;
+
+	case TYPE_INT:
+		val->val.int_val = rand();
+		break;
+
+	case TYPE_TIME:
+		val->val.double_val = CurrTime();
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_INTERVAL:
+		val->val.double_val = random();
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = rand();
+		break;
+
+	case TYPE_PORT:
+		val->val.port_val.port = rand() / (RAND_MAX / 60000);
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		break;
+
+	case TYPE_SUBNET: 
+		{
+		val->val.subnet_val.prefix = StringToAddr("192.168.17.1");
+		val->val.subnet_val.length = 16;
+		}
+		break;
+
+	case TYPE_ADDR: 
+		val->val.addr_val = StringToAddr("192.168.17.1");
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		// First - common initialization
+		// Then - initialization for table.
+		// Then - initialization for vector.
+		// Then - common stuff
+		{
+		// how many entries do we have...
+		unsigned int length = rand() / (RAND_MAX / 15);
+
+		Value** lvals = new Value* [length];
+
+		if ( type == TYPE_TABLE ) 
+			{
+			val->val.set_val.vals = lvals;
+			val->val.set_val.size = length;
+			} 
+		else if ( type == TYPE_VECTOR ) 
+			{
+			val->val.vector_val.vals = lvals;
+			val->val.vector_val.size = length;
+			} 
+		else 
+			assert(false);
+
+		if ( length == 0 )
+			break; //empty
+
+		for ( unsigned int pos = 0; pos < length; pos++ ) 
+			{
+			Value* newval = EntryToVal(subtype, TYPE_ENUM);
+			if ( newval == 0 ) 
+				{
+				Error("Error while reading set");
+				return 0;
+				}
+			lvals[pos] = newval;
+			}
+
+		break;
+		}
+
+
+	default:
+		Error(Fmt("unsupported field format %d", type));
+		return 0;
+	}	
+
+	return val;
+
+	}
+
+
+bool Benchmark::DoHeartbeat(double network_time, double current_time)
+{
+	ReaderBackend::DoHeartbeat(network_time, current_time);
+	num_lines = (int) ( (double) num_lines*multiplication_factor);
+	num_lines += add;
+	heartbeatstarttime = CurrTime();
+
+	switch ( mode ) {
+		case MANUAL:
+			// yay, we do nothing :)
+			break;
+		case REREAD:
+		case STREAM:
+			if ( multiplication_factor != 1 || add != 0 ) 
+				{
+				// we have to document at what time we changed the factor to what value.
+				Value** v = new Value*[2];
+				v[0] = new Value(TYPE_COUNT, true);
+				v[0]->val.uint_val = num_lines;
+				v[1] = new Value(TYPE_TIME, true);
+				v[1]->val.double_val = CurrTime();
+
+				SendEvent("lines_changed", 2, v);
+				}
+
+			if ( autospread != 0.0 ) 
+				// because executing this in every loop is apparently too expensive.
+				autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
+	
+			Update(); // call update and not DoUpdate, because update actually checks disabled.
+
+			SendEvent("HeartbeatDone", 0, 0);
+			break;
+		default:
+			assert(false);
+	}
+
+	return true;
+}
+

src/input/readers/Benchmark.h

@@ -0,0 +1,59 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef INPUT_READERS_BENCHMARK_H
+#define INPUT_READERS_BENCHMARK_H
+
+
+#include "../ReaderBackend.h"
+
+namespace input { namespace reader {
+
+class Benchmark : public ReaderBackend {
+public:
+    Benchmark(ReaderFrontend* frontend);
+    ~Benchmark();
+    
+    static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); }
+    
+protected:
+	
+	virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields);
+
+	virtual void DoClose();
+
+	virtual bool DoUpdate();
+
+private:
+
+	virtual bool DoHeartbeat(double network_time, double current_time);
+
+	unsigned int num_fields;
+
+	double CurrTime();
+
+	const threading::Field* const * fields; // raw mapping		
+	
+	threading::Value* EntryToVal(TypeTag Type, TypeTag subtype);
+
+	int mode;
+	int num_lines;
+
+	double multiplication_factor;
+	int spread;
+	double autospread;
+	int autospread_time;
+	int add;
+	int stopspreadat;
+	double heartbeatstarttime;
+	double timedspread;
+	double heart_beat_interval;
+
+	string RandomString(const int len);
+
+};
+
+
+}
+}
+
+#endif /* INPUT_READERS_BENCHMARK_H */

src/input/readers/Raw.cc

@@ -0,0 +1,282 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Raw.h"
+#include "NetVar.h"
+
+#include <fstream>
+#include <sstream>
+
+#include "../../threading/SerialTypes.h"
+#include "../fdstream.h"
+
+#define MANUAL 0
+#define REREAD 1
+#define STREAM 2
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <fcntl.h>
+
+using namespace input::reader;
+using threading::Value;
+using threading::Field;
+
+Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend)
+	{
+	file = 0;
+	in = 0;
+
+	separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len());
+	if ( separator.size() != 1 ) 
+		Error("separator length has to be 1. Separator will be truncated.");
+	}
+
+Raw::~Raw()
+	{
+	DoClose();
+	}
+
+void Raw::DoClose()
+	{
+	if ( file != 0 ) 
+		{
+		Close();
+		}
+	}
+
+bool Raw::Open() 
+	{
+	if ( execute ) 
+		{
+		file = popen(fname.c_str(), "r");
+		if ( file == NULL ) 
+			{
+			Error(Fmt("Could not execute command %s", fname.c_str()));
+			return false;
+			}
+		}
+	else 
+		{
+		file = fopen(fname.c_str(), "r");
+		if ( file == NULL ) 
+			{
+			Error(Fmt("Init: cannot open %s", fname.c_str()));
+			return false;
+			}
+		}
+	
+	in = new boost::fdistream(fileno(file));
+
+	if ( execute && mode == STREAM ) 
+		{
+		fcntl(fileno(file), F_SETFL, O_NONBLOCK);
+		}
+
+	return true;
+	}
+
+bool Raw::Close()
+	{
+	if ( file == NULL ) 
+		{
+		InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str()));
+		return false;
+		}
+
+	if ( execute ) 
+		{
+		delete(in);
+		pclose(file);
+		} 
+	else 
+		{
+		delete(in);
+		fclose(file);
+		}
+
+	in = NULL;
+	file = NULL;
+
+	return true;
+	}
+
+bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields)
+	{
+	fname = path;
+	mode = arg_mode;
+	mtime = 0;
+	execute = false;
+	firstrun = true;
+	bool result;
+
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	if ( path.length() == 0 ) 
+		{
+		Error("No source path provided");
+		return false;
+		}
+	
+	if ( arg_num_fields != 1 ) 
+		{
+		Error("Filter for raw reader contains more than one field. "
+		      "Filters for the raw reader may only contain exactly one string field. "
+		      "Filter ignored.");
+		return false;
+		}
+
+	if ( fields[0]->type != TYPE_STRING ) 
+		{
+		Error("Filter for raw reader contains a field that is not of type string.");
+		return false;
+		}
+
+	// do Initialization
+	char last = path[path.length()-1];
+	if ( last == '|' ) 
+		{
+		execute = true;
+		fname = path.substr(0, fname.length() - 1);
+
+		if ( ( mode != MANUAL ) && ( mode != STREAM ) ) {
+			Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str()));
+			return false;
+		} 	
+		
+		result = Open();
+
+	} else {
+		execute = false;
+		if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) 
+			{
+			Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str()));
+			return false;
+			}
+
+		result = Open();	
+		}
+
+	if ( result == false ) 
+		return result;
+
+#ifdef DEBUG
+	Debug(DBG_INPUT, "Raw reader created, will perform first update");
+#endif
+
+	// after initialization - do update
+	DoUpdate();
+
+#ifdef DEBUG
+	Debug(DBG_INPUT, "First update went through");
+#endif
+	return true;
+	}
+
+
+bool Raw::GetLine(string& str) 
+	{
+	if ( in->peek() == std::iostream::traits_type::eof() ) 
+		return false;
+
+	if ( in->eofbit == true || in->failbit == true ) 
+		return false;
+
+	while ( getline(*in, str, separator[0]) ) 
+		return true;
+
+	return false;
+	}
+
+
+// read the entire file and send appropriate thingies back to InputMgr
+bool Raw::DoUpdate() 
+	{
+	if ( firstrun ) 
+		firstrun = false;
+	else
+		{
+		switch ( mode ) {
+			case REREAD:
+				{
+				// check if the file has changed
+				struct stat sb;
+				if ( stat(fname.c_str(), &sb) == -1 ) 
+					{
+					Error(Fmt("Could not get stat for %s", fname.c_str()));
+					return false;
+					}
+
+				if ( sb.st_mtime <= mtime ) 
+					// no change
+					return true;
+
+				mtime = sb.st_mtime;
+				// file changed. reread.
+
+				// fallthrough
+				}
+			case MANUAL:
+			case STREAM:
+				if ( mode == STREAM && file != NULL && in != NULL ) 
+					{
+					//fpurge(file);
+					in->clear(); // remove end of file evil bits
+					break;
+					}
+
+				Close();
+				if ( !Open() ) 
+					return false;
+
+				break;
+			default:
+				assert(false);
+
+		}
+		}
+
+	string line;
+	while ( GetLine(line) ) 
+		{
+		assert (num_fields == 1);
+	
+		Value** fields = new Value*[1];
+
+		// filter has exactly one text field. convert to it.
+		Value* val = new Value(TYPE_STRING, true);
+		val->val.string_val = new string(line);
+		fields[0] = val;
+		
+		Put(fields);
+		}
+
+#ifdef DEBUG
+	Debug(DBG_INPUT, "DoUpdate finished successfully");
+#endif
+
+	return true;
+	}
+
+
+bool Raw::DoHeartbeat(double network_time, double current_time)
+	{
+	ReaderBackend::DoHeartbeat(network_time, current_time);
+
+	switch ( mode ) {
+		case MANUAL:
+			// yay, we do nothing :)
+			break;
+		case REREAD:
+		case STREAM:
+			Update(); // call update and not DoUpdate, because update 
+		       	          // checks disabled.
+			break;
+		default:
+			assert(false);
+	}
+
+	return true;
+	}

src/input/readers/Raw.h

@@ -0,0 +1,64 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef INPUT_READERS_RAW_H
+#define INPUT_READERS_RAW_H
+
+#include <iostream>
+#include <vector>
+
+#include "../ReaderBackend.h"
+
+namespace input { namespace reader {
+
+class Raw : public ReaderBackend {
+public:
+    Raw(ReaderFrontend* frontend);
+    ~Raw();
+    
+    static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); }
+    
+protected:
+	
+	virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields);
+
+	virtual void DoClose();
+
+	virtual bool DoUpdate();
+
+private:
+
+	virtual bool DoHeartbeat(double network_time, double current_time);
+	bool Open();
+	bool Close();
+
+	bool GetLine(string& str);
+	
+	istream* in;
+
+	FILE* file;
+	
+	string fname;
+
+	// Options set from the script-level.
+	string separator;
+
+	// keep a copy of the headerline to determine field locations when filters change
+	string headerline;
+
+	int mode;
+	bool execute;
+	bool firstrun;
+
+	time_t mtime;
+	
+	unsigned int num_fields;
+
+	const threading::Field* const * fields; // raw mapping		
+
+};
+
+
+}
+}
+
+#endif /* INPUT_READERS_RAW_H */

src/logging/Manager.cc

@@ -477,6 +477,7 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
 		threading::Field* field = new threading::Field();
 		field->name = new_path;
 		field->type = t->Tag();
+		field->optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
 
 		if ( field->type == TYPE_TABLE )
 			field->subtype = t->AsSetType()->Indices()->PureType()->Tag();

src/main.cc

@@ -50,6 +50,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "Brofiler.h"
 
 #include "threading/Manager.h"
+#include "input/Manager.h"
 #include "logging/Manager.h"
 #include "logging/writers/Ascii.h"
 
@@ -80,6 +81,7 @@ DNS_Mgr* dns_mgr;
 TimerMgr* timer_mgr;
 logging::Manager* log_mgr = 0;
 threading::Manager* thread_mgr = 0;
+input::Manager* input_mgr = 0;
 Stmt* stmts;
 EventHandlerPtr net_done = 0;
 RuleMatcher* rule_matcher = 0;
@@ -314,6 +316,8 @@ void terminate_bro()
 	log_mgr->Terminate();
 	thread_mgr->Terminate();
 
+	mgr.Drain();
+
 	delete timer_mgr;
 	delete dns_mgr;
 	delete persistence_serializer;
@@ -763,6 +767,7 @@ int main(int argc, char** argv)
 	remote_serializer = new RemoteSerializer();
 	event_registry = new EventRegistry();
 	log_mgr = new logging::Manager();
+    	input_mgr = new input::Manager();
 
 	if ( events_file )
 		event_player = new EventPlayer(events_file);

src/parse.y

@@ -2,7 +2,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 87
+%expect 90
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -24,6 +24,7 @@
 %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED
 %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE
 %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER
+%token TOK_ATTR_TYPE_COLUMN
 
 %token TOK_DEBUG
 
@@ -1306,6 +1307,8 @@ attr:
 			{ $$ = new Attr(ATTR_PRIORITY, $3); }
 	|	TOK_ATTR_GROUP '=' expr
 			{ $$ = new Attr(ATTR_GROUP, $3); }
+	|	TOK_ATTR_TYPE_COLUMN '=' expr
+			{ $$ = new Attr(ATTR_TYPE_COLUMN, $3); }
 	|	TOK_ATTR_LOG
 			{ $$ = new Attr(ATTR_LOG); }
 	|	TOK_ATTR_ERROR_HANDLER

src/scan.l

@@ -331,6 +331,7 @@ when	return TOK_WHEN;
 &optional	return TOK_ATTR_OPTIONAL;
 &persistent	return TOK_ATTR_PERSISTENT;
 &priority	return TOK_ATTR_PRIORITY;
+&type_column	return TOK_ATTR_TYPE_COLUMN;
 &read_expire	return TOK_ATTR_EXPIRE_READ;
 &redef		return TOK_ATTR_REDEF;
 &rotate_interval	return TOK_ATTR_ROTATE_INTERVAL;

src/threading/Manager.cc

@@ -1,5 +1,6 @@
 
 #include "Manager.h"
+#include "NetVar.h"
 
 using namespace threading;
 
@@ -11,6 +12,9 @@ Manager::Manager()
 	next_beat = 0;
 	terminating = false;
 	idle = true;
+
+	heart_beat_interval = double(BifConst::Threading::heart_beat_interval);
+	DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval);
 	}
 
 Manager::~Manager()
@@ -57,6 +61,12 @@ void Manager::KillThreads()
 
 void Manager::AddThread(BasicThread* thread)
 	{
+	if ( heart_beat_interval == 0 ) {
+		// Sometimes initialization does not seem to work from constructor.
+		heart_beat_interval = double(BifConst::Threading::heart_beat_interval);
+		DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval);
+	}
+
 	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
 	all_threads.push_back(thread);
 	idle = false;
@@ -81,6 +91,12 @@ double Manager::NextTimestamp(double* network_time)
 		// is due or not set yet), we want to check for more asap.
 		return timer_mgr->Time();
 
+	for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ )
+		{
+			if ( (*i)->MightHaveOut() )
+				return timer_mgr->Time();
+		}
+
 	return -1.0;
 	}
 
@@ -91,7 +107,7 @@ void Manager::Process()
 	if ( network_time && (network_time > next_beat || ! next_beat) )
 		{
 		do_beat = true;
-		next_beat = ::network_time + HEART_BEAT_INTERVAL;
+		next_beat = ::network_time + heart_beat_interval;
 		}
 
 	did_process = false;

src/threading/Manager.h

@@ -44,7 +44,7 @@ public:
 	void Terminate();
 
 	/**
-	 * Returns True if we are currently in Terminate() waiting for 
+	 * Returns True if we are currently in Terminate() waiting for
 	 * threads to exit.
 	 */
 	bool Terminating() const	{ return terminating; }
@@ -99,7 +99,7 @@ protected:
 	 * Registers a new message thread with the manager. This is
 	 * automatically called by the thread's constructor. This must be
 	 * called \a in \a addition to AddThread(BasicThread* thread). The
-	 * MsgThread constructor makes sure to do so. 
+	 * MsgThread constructor makes sure to do so.
 	 *
 	 * @param thread The thread.
 	 */
@@ -126,7 +126,7 @@ protected:
 	virtual const char* Tag()	{ return "threading::Manager"; }
 
 private:
-	static const int HEART_BEAT_INTERVAL = 1;
+	int heart_beat_interval;
 
 	typedef std::list<BasicThread*> all_thread_list;
 	all_thread_list all_threads;

src/threading/MsgThread.cc

@@ -301,7 +301,7 @@ void MsgThread::Run()
 
 		if ( ! result )
 			{
-			string s = msg->Name() + " failed, terminating thread";
+			string s = msg->Name() + " failed, terminating thread (MsgThread)";
 			Error(s.c_str());
 			Stop();
 			break;

src/threading/MsgThread.h

@@ -273,6 +273,13 @@ private:
 	 */
 	bool HasOut()	{ return queue_out.Ready(); }
 
+	/**
+	 * Returns true if there might be at least one message pending for
+	 * the main thread. This function may occasionally return a value not
+	 * indicating the actual state, but won't do so very often.
+	 */
+	bool MightHaveOut() { return queue_out.MaybeReady(); }
+
 	Queue<BasicInputMessage *> queue_in;
 	Queue<BasicOutputMessage *> queue_out;
 

src/threading/Queue.h

@@ -53,6 +53,13 @@ public:
 	 */
 	bool Ready();
 
+	/**
+	 * Returns true if the next Get() operation might succeed.
+	 * This function may occasionally return a value not
+	 * indicating the actual state, but won't do so very often.
+	 */
+	bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); }
+
 	/**
 	 * Returns the number of queued items not yet retrieved.
 	 */

src/threading/SerialTypes.cc

@@ -12,7 +12,12 @@ bool Field::Read(SerializationFormat* fmt)
 	int t;
 	int st;
 
-	bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") );
+	bool success = (fmt->Read(&name, "name")
+			&& fmt->Read(&secondary_name, "secondary_name")
+			&& fmt->Read(&t, "type")
+			&& fmt->Read(&st, "subtype")
+			&& fmt->Read(&optional, "optional"));
+
 	type = (TypeTag) t;
 	subtype = (TypeTag) st;
 
@@ -21,7 +26,11 @@ bool Field::Read(SerializationFormat* fmt)
 
 bool Field::Write(SerializationFormat* fmt) const
 	{
-	return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
+	return (fmt->Write(name, "name")
+		&& fmt->Write(secondary_name, "secondary_name")
+		&& fmt->Write((int)type, "type")
+		&& fmt->Write((int)subtype, "subtype"),
+		fmt->Write(optional, "optional"));
 	}
 
 string Field::TypeName() const
@@ -186,7 +195,7 @@ bool Value::Read(SerializationFormat* fmt)
 		char length;
 		char family;
 
-		if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) 
+		if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) )
 			return false;
 
 		switch ( family ) {

src/threading/SerialTypes.h

@@ -20,19 +20,23 @@ namespace threading {
  */
 struct Field {
 	string name;	//! Name of the field.
+	//! Needed by input framework. Port fields have two names (one for the
+	//! port, one for the type), and this specifies the secondary name.
+	string secondary_name;
 	TypeTag type;	//! Type of the field.
 	TypeTag subtype;	//! Inner type for sets.
+	bool optional;	//! True if field is optional.
 
 	/**
 	 * Constructor.
 	 */
-	Field() 	{ subtype = TYPE_VOID; }
+	Field() 	{ subtype = TYPE_VOID; optional = false; }
 
 	/**
 	 * Copy constructor.
 	 */
 	Field(const Field& other)
-		: name(other.name), type(other.type), subtype(other.subtype) {  }
+		: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) {  }
 
 	/**
 	 * Unserializes a field.

src/types.bif

@@ -169,4 +169,25 @@ enum ID %{
 	Unknown,
 %}
 
+module Input;
+
+enum Reader %{
+	READER_DEFAULT,
+	READER_ASCII,
+	READER_RAW,
+	READER_BENCHMARK,
+%}
+
+enum Event %{
+	EVENT_NEW,
+	EVENT_CHANGED,
+	EVENT_REMOVED,
+%}
+
+enum Mode %{
+	MANUAL = 0,
+	REREAD = 1,
+	STREAM = 2,
+%}
+
 module GLOBAL;

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -20,4 +20,8 @@ scripts/base/init-bare.bro
       scripts/base/frameworks/logging/./postprocessors/./sftp.bro
     scripts/base/frameworks/logging/./writers/ascii.bro
     scripts/base/frameworks/logging/./writers/dataseries.bro
+  scripts/base/frameworks/input/__load__.bro
+    scripts/base/frameworks/input/./main.bro
+      build/src/base/input.bif.bro
+    scripts/base/frameworks/input/./readers/ascii.bro
 scripts/policy/misc/loaded-scripts.bro

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -20,6 +20,10 @@ scripts/base/init-bare.bro
       scripts/base/frameworks/logging/./postprocessors/./sftp.bro
     scripts/base/frameworks/logging/./writers/ascii.bro
     scripts/base/frameworks/logging/./writers/dataseries.bro
+  scripts/base/frameworks/input/__load__.bro
+    scripts/base/frameworks/input/./main.bro
+      build/src/base/input.bif.bro
+    scripts/base/frameworks/input/./readers/ascii.bro
 scripts/base/init-default.bro
   scripts/base/utils/site.bro
     scripts/base/utils/./patterns.bro

testing/btest/Baseline/scripts.base.frameworks.input.basic/out

@@ -0,0 +1,14 @@
+{
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}

testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out

@@ -0,0 +1,4 @@
+{
+[2] = [b=<uninitialized>],
+[1] = [b=T]
+}

testing/btest/Baseline/scripts.base.frameworks.input.event/out

@@ -0,0 +1,70 @@
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+1
+T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+2
+T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+3
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+4
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+5
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+6
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
+Input::EVENT_NEW
+7
+T

testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out

@@ -0,0 +1,9 @@
+[source=wc -l ../input.log |, reader=Input::READER_RAW, mode=Input::MANUAL, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print outfile, description;
+print outfile, tpe;
+print outfile, s;
+close(outfile);
+}]
+Input::EVENT_NEW
+       8 ../input.log

testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out

@@ -0,0 +1,145 @@
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+dfsdf
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdf
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+
+done

testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out

@@ -0,0 +1,3 @@
+{
+[-42] = T
+}

testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out

@@ -0,0 +1,3 @@
+{
+[-42] = [b=T]
+}

testing/btest/Baseline/scripts.base.frameworks.input.optional/out

@@ -0,0 +1,9 @@
+{
+[2] = [b=T, notb=F],
+[4] = [b=F, notb=T],
+[6] = [b=F, notb=T],
+[7] = [b=T, notb=F],
+[1] = [b=T, notb=F],
+[5] = [b=F, notb=T],
+[3] = [b=F, notb=T]
+}

testing/btest/Baseline/scripts.base.frameworks.input.port/out

@@ -0,0 +1,3 @@
+[p=80/tcp]
+[p=52/udp]
+[p=30/unknown]

testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out

@@ -0,0 +1,7 @@
+VALID
+VALID
+VALID
+VALID
+VALID
+VALID
+VALID

testing/btest/Baseline/scripts.base.frameworks.input.predicate/out

@@ -0,0 +1,7 @@
+VALID
+VALID
+VALID
+VALID
+VALID
+VALID
+VALID

testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out

@@ -0,0 +1,4 @@
+{
+[2, idxmodified] = [b=T, s=test2],
+[1, idx1] = [b=T, s=testmodified]
+}

testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out

@@ -0,0 +1,23 @@
+Update_finished for input, try 1
+{
+[2, idxmodified] = [b=T, s=test2],
+[1, idx1] = [b=T, s=testmodified]
+}
+Update_finished for input, try 2
+{
+[2, idxmodified] = [b=T, s=test2],
+[1, idx1] = [b=F, s=testmodified]
+}
+Update_finished for input, try 3
+{
+[2, idxmodified] = [b=F, s=test2],
+[1, idx1] = [b=F, s=testmodified]
+}
+Update_finished for input, try 4
+{
+[2, idxmodified] = [b=F, s=test2]
+}
+Update_finished for input, try 5
+{
+[1, idx1] = [b=T, s=testmodified]
+}

testing/btest/Baseline/scripts.base.frameworks.input.raw/out

@@ -0,0 +1,64 @@
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+dfsdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.

testing/btest/Baseline/scripts.base.frameworks.input.repeat/out

@@ -0,0 +1,160 @@
+input0
+input.log
+{
+[1] = T
+}
+input1
+input.log
+{
+[1] = T
+}
+input2
+input.log
+{
+[1] = T
+}
+input3
+input.log
+{
+[1] = T
+}
+input4
+input.log
+{
+[1] = T
+}
+input5
+input.log
+{
+[1] = T
+}
+input6
+input.log
+{
+[1] = T
+}
+input7
+input.log
+{
+[1] = T
+}
+input8
+input.log
+{
+[1] = T
+}
+input9
+input.log
+{
+[1] = T
+}
+input10
+input.log
+{
+[1] = T
+}
+input11
+input.log
+{
+[1] = T
+}
+input12
+input.log
+{
+[1] = T
+}
+input13
+input.log
+{
+[1] = T
+}
+input14
+input.log
+{
+[1] = T
+}
+input15
+input.log
+{
+[1] = T
+}
+input16
+input.log
+{
+[1] = T
+}
+input17
+input.log
+{
+[1] = T
+}
+input18
+input.log
+{
+[1] = T
+}
+input19
+input.log
+{
+[1] = T
+}
+input20
+input.log
+{
+[1] = T
+}
+input21
+input.log
+{
+[1] = T
+}
+input22
+input.log
+{
+[1] = T
+}
+input23
+input.log
+{
+[1] = T
+}
+input24
+input.log
+{
+[1] = T
+}
+input25
+input.log
+{
+[1] = T
+}
+input26
+input.log
+{
+[1] = T
+}
+input27
+input.log
+{
+[1] = T
+}
+input28
+input.log
+{
+[1] = T
+}
+input29
+input.log
+{
+[1] = T
+}
+input30
+input.log
+{
+[1] = T
+}
+input31
+input.log
+{
+[1] = T
+}

testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out

@@ -0,0 +1,128 @@
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+dfsdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+dfsdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.

testing/btest/Baseline/scripts.base.frameworks.input.stream/out

@@ -0,0 +1,115 @@
+============EVENT============
+Input::EVENT_NEW
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============SERVERS============
+{
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+============EVENT============
+Input::EVENT_NEW
+[i=-43]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+============EVENT============
+Input::EVENT_CHANGED
+[i=-43]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============SERVERS============
+{
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+done

testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out

@@ -0,0 +1,120 @@
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+dfsdf
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdf
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.

testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out

@@ -0,0 +1,126 @@
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=1]
+T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=2]
+T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=3]
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=4]
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=5]
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=6]
+F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
+Input::EVENT_NEW
+[i=7]
+T

testing/btest/Baseline/scripts.base.frameworks.input.twotables/out

@@ -0,0 +1,349 @@
+============PREDICATE============
+Input::EVENT_NEW
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE 2============
+Input::EVENT_NEW
+[i=-43]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=-42]
+Right
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+==========SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+============EVENT============
+Description
+[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh2, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE 2============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=-43]
+Right
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+==========SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+============PREDICATE============
+Input::EVENT_NEW
+[i=-44]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_REMOVED
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=-44]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+Input::EVENT_REMOVED
+Type
+[i=-42]
+Left
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+Right
+==========SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}

testing/btest/btest.cfg

@@ -3,7 +3,7 @@ TestDirs    = doc bifs language core scripts istate coverage
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
-IgnoreFiles = *.tmp *.swp #* *.trace
+IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
 
 [environment]
 BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev`

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -0,0 +1,50 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+}
+
+event Input::update_finished(name: string, source:string) {
+	print servers;
+}

testing/btest/scripts/base/frameworks/input/emptyvals.bro

@@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i
+##types	bool	int
+T	1
+-	2
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+}
+
+event Input::update_finished(name: string, source:string) {
+	print servers;
+}

testing/btest/scripts/base/frameworks/input/event.bro

@@ -0,0 +1,38 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+
+module A;
+
+type Val: record {
+	i: int;
+	b: bool;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) {
+	print description;
+	print tpe;
+	print i;
+	print b;
+}
+
+event bro_init()
+{
+	Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/executeraw.bro

@@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 1
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	close(outfile);
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/executestreamraw.bro

@@ -0,0 +1,58 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	try = try + 1;
+	
+	if ( try == 9 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+}

testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro

@@ -0,0 +1,36 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	
+#types	bool	int
+T	-42	
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers;
+}
+

testing/btest/scripts/base/frameworks/input/onecolumn-record.bro

@@ -0,0 +1,36 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	
+#types	bool	int
+T	-42	
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	Input::add_table([$name="input", $source="input.log", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers;
+}
+

testing/btest/scripts/base/frameworks/input/optional.bro

@@ -0,0 +1,45 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	notb: bool &optional;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
+				$pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; }
+				]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers;
+}

testing/btest/scripts/base/frameworks/input/port.bro

@@ -0,0 +1,40 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#fields	i	p	t
+1.2.3.4	80	tcp
+1.2.3.5	52	udp
+1.2.3.6	30	unknown
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: addr;
+};
+
+type Val: record {
+	p: port &type_column="t";
+};
+
+global servers: table[addr] of Val = table();
+
+event bro_init()
+{
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]);
+	print servers[1.2.3.4];
+	print servers[1.2.3.5];
+	print servers[1.2.3.6];
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers[1.2.3.4];
+	print servers[1.2.3.5];
+	print servers[1.2.3.6];
+}
+

testing/btest/scripts/base/frameworks/input/predicate-stream.bro

@@ -0,0 +1,80 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+#
+# only difference from predicate.bro is, that this one uses a stream source.
+# the reason is, that the code-paths are quite different, because then the ascii reader uses the put and not the sendevent interface
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+global ct: int;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) {
+	ct = ct + 1;
+	if ( ct < 3 ) {
+		return;
+	}
+	if ( ct > 3 ) {
+		print "Too many events";
+		return;
+	}
+
+	if ( 1 in servers ) {
+		print "VALID";
+	}
+	if ( 2 in servers ) {
+		print "VALID";
+	}
+	if ( !(3 in servers) ) {
+		print "VALID";
+	}
+	if ( !(4 in servers) ) {
+		print "VALID";
+	}
+	if ( !(5 in servers) ) {
+		print "VALID";
+	}
+	if ( !(6 in servers) ) {
+		print "VALID";
+	}
+	if ( 7 in servers ) {
+		print "VALID";
+	}
+}
+
+event bro_init()
+{
+	ct = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line,
+				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
+				]);
+	Input::remove("input");
+
+}
+

testing/btest/scripts/base/frameworks/input/predicate.bro

@@ -0,0 +1,64 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F,
+				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
+				]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	if ( 1 in servers ) {
+		print "VALID";
+	}
+	if ( 2 in servers ) {
+		print "VALID";
+	}
+	if ( !(3 in servers) ) {
+		print "VALID";
+	}
+	if ( !(4 in servers) ) {
+		print "VALID";
+	}
+	if ( !(5 in servers) ) {
+		print "VALID";
+	}
+	if ( !(6 in servers) ) {
+		print "VALID";
+	}
+	if ( 7 in servers ) {
+		print "VALID";
+	}
+}

testing/btest/scripts/base/frameworks/input/predicatemodify.bro

@@ -0,0 +1,50 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	T	test1	idx1
+2	T	test2	idx2
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+	ss: string;
+};
+
+type Val: record {
+	b: bool;
+	s: string;
+};
+
+global servers: table[int, string] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
+				$pred(typ: Input::Event, left: Idx, right: Val) = { 
+				if ( left$i == 1 ) {
+					right$s = "testmodified";
+				} 
+
+				if ( left$i == 2 ) {
+					left$ss = "idxmodified";
+				} 
+				return T; 
+				}
+				]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers;
+}

testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro

@@ -0,0 +1,107 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro %INPUT 
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input4.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input5.log input.log
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff out
+#
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	T	test1	idx1
+2	T	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	F	test1	idx1
+2	T	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	F	test1	idx1
+2	F	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input4.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+2	F	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input5.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	T	test1	idx1
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+	ss: string;
+};
+
+type Val: record {
+	b: bool;
+	s: string;
+};
+
+global servers: table[int, string] of Val = table();
+global outfile: file;
+global try: count;
+
+event bro_init()
+{
+	try = 0;
+	outfile = open ("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $mode=Input::REREAD,
+				$pred(typ: Input::Event, left: Idx, right: Val) = { 
+				if ( left$i == 1 ) {
+					right$s = "testmodified";
+				} 
+
+				if ( left$i == 2 ) {
+					left$ss = "idxmodified";
+				} 
+				return T; 
+				}
+				]);
+}
+
+event Input::update_finished(name: string, source: string) {
+	try = try + 1;
+	print outfile, fmt("Update_finished for %s, try %d", name, try);
+	print outfile, servers;
+	
+	if ( try == 5 ) {
+		close (outfile);
+		Input::remove("input");
+	}
+}

testing/btest/scripts/base/frameworks/input/raw.bro

@@ -0,0 +1,33 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print description;
+	print tpe;
+	print s;
+}
+
+event bro_init()
+{
+	Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/repeat.bro

@@ -0,0 +1,41 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global destination: table[int] of Val = table();
+
+const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32};
+
+event bro_init()
+{
+	for ( i in one_to_32 ) {
+		Input::add_table([$source="input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]);
+		Input::remove(fmt("input%d", i));
+	}
+}
+
+event Input::update_finished(name: string, source: string) {
+	print name;
+	print source;
+	print destination;
+}

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -0,0 +1,132 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro %INPUT 
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input4.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input5.log input.log
+# @TEST-EXEC: btest-bg-wait -k 2
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input4.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-44	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-45	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-46	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-47	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-48	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input5.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+F	-48	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+}
+
+
+event Input::update_finished(name: string, source: string) {
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 5 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}

testing/btest/scripts/base/frameworks/input/rereadraw.bro

@@ -0,0 +1,34 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print description;
+	print tpe;
+	print s;
+}
+
+event bro_init()
+{
+	Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]);
+	Input::force_update("input");
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/stream.bro

@@ -0,0 +1,83 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+	print outfile, "============EVENT============";
+	print outfile, tpe;
+	print outfile, left;
+	print outfile, right;
+	print outfile, "============SERVERS============";
+	print outfile, servers;
+
+	try = try + 1;
+	
+	if ( try == 3 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]);
+}
+

testing/btest/scripts/base/frameworks/input/streamraw.bro

@@ -0,0 +1,56 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	
+	if ( try == 3 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+}

testing/btest/scripts/base/frameworks/input/tableevent.bro

@@ -0,0 +1,42 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global destination: table[int] of Val = table();
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) {
+	print description;
+	print tpe;
+	print left;
+	print right;
+}
+
+event bro_init()
+{
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]);
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/twotables.bro

@@ -0,0 +1,113 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro %INPUT 
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: btest-bg-wait -k 2
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+F	-44	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE 2============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+}
+
+
+event Input::update_finished(name: string, source: string) {
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 5 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}