Merge remote-tracking branch 'origin/topic/bernhard/input-threads' into topic/bernhard/input-threads

doc/input.rst

@@ -34,9 +34,11 @@ very similar to the abstracts used in the logging framework:
 
         Readers
                 A reader defines the input format for the specific input stream.
-                At the moment, Bro comes with only one type of reader, which can
+                At the moment, Bro comes with two types of reader. The default reader is READER_ASCII,
-                read the tab seperated ASCII logfiles that were generated by the
+                which can read the tab seperated ASCII logfiles that were generated by the
                 logging framework.
+                READER_RAW can files containing records separated by a character(like e.g. newline) and send
+                one event per line.
 
 
 Basics
@@ -69,6 +71,20 @@ The fields that can be set when creating a stream are:
         ``reader``
                 The reader used for this stream. Default is ``READER_ASCII``.
 
+        ``mode``
+                The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``.
+                Default is ``MANUAL``.
+                ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not
+                be reflected in the data bro knows.
+                ``REREAD`` means that the whole file is read again each time a change is found. This should be used for 
+                files that are mapped to a table where individual lines can change.
+                ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new
+                data is added to the file.
+
+        ``autostart``
+                If set to yes, the first update operation is triggered automatically after the first filter has been added to the stream.
+                This has to be set to false if several filters are added to the input source.
+                In this case Input::force_update has to be called manually once after all filters have been added.       
 
 Filters
 =======
@@ -101,9 +117,6 @@ could be defined as follows:
                 ...
 
                 Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]);
-
-                # read the file after all filters have been set
-                Input::force_update(Foo::INPUT); 
         }
 
 The fields that can be set for an event filter are:
@@ -156,7 +169,7 @@ an approach similar to this:
 
                 Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]);
 
-                # read the file after all filters have been set
+                # read the file after all filters have been set (only needed if autostart is set to false)
                 Input::force_update(Foo::INPUT);
         }
 

scripts/base/frameworks/input/main.bro

@@ -5,15 +5,15 @@ module Input;
 
 export {
          
-	redef enum Input::ID += { TABLE_READ };
-
 	## The default input reader used. Defaults to `READER_ASCII`.
 	const default_reader = READER_ASCII &redef;
 
 	const default_mode = MANUAL &redef;
 
-	## Stream decription type used for the `create_stream` method
+	## TableFilter description type used for the `table` method.
-	type StreamDescription: record {
+	type TableDescription: record {
+		## Common definitions for tables and events
+		
 		## String that allows the reader to find the source.
 		## For `READER_ASCII`, this is the filename.
 		source: string;
@@ -26,13 +26,12 @@ export {
 
 		## Automatically start the input stream after the first filter has been added
 		autostart: bool &default=T;
-	};
 
-	## TableFilter description type used for the `add_tablefilter` method.
-	type TableFilter: record {
 		## Descriptive name. Used to remove a filter at a later time
 		name: string; 
 
+		## Special definitions for tables
+
 		## Table which will contain the data read by the input framework
 		destination: any;
 		## Record that defines the values used as the index of the table
@@ -55,11 +54,28 @@ export {
 		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
 	};
 
-	## EventFilter description type used for the `add_eventfilter` method.
+	## EventFilter description type used for the `event` method.
-	type EventFilter: record {
+	type EventDescription: record {
+		## Common definitions for tables and events
+		
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+		
+		## Reader to use for this steam
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Automatically start the input stream after the first filter has been added
+		autostart: bool &default=T;
+
 		## Descriptive name. Used to remove a filter at a later time
 		name: string; 
 
+		## Special definitions for events
+	
 		## Record describing the fields to be retrieved from the source input.
 		fields: any;
 		## If want_record if false (default), the event receives each value in fields as a seperate argument.
@@ -72,59 +88,29 @@ export {
 
 	};
 
-	#const no_filter: Filter = [$name="<not found>", $idx="", $val="", $destination=""]; # Sentinel.
+	## Create a new table input from a given source. Returns true on success.
-
-	## Create a new input stream from a given source. Returns true on success.
 	##
-	## id: `Input::ID` enum value identifying this stream
+	## description: `TableDescription` record describing the source.
-	## description: `StreamDescription` record describing the source.
+	global add_table: function(description: Input::TableDescription) : bool;
-	global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool;
 	
-	## Remove a current input stream. Returns true on success.
+	## Create a new event input from a given source. Returns true on success.
 	##
-	## id: `Input::ID` enum value identifying the stream to be removed
+	## description: `TableDescription` record describing the source.
-	global remove_stream: function(id: Input::ID) : bool;
+	global add_event: function(description: Input::EventDescription) : bool;
+
+	## Remove a input stream. Returns true on success and false if the named stream was not found.
+	##
+	## id: string value identifying the stream to be removed
+	global remove: function(id: string) : bool;
 
 	## Forces the current input to be checked for changes.
+	## Returns true on success and false if the named stream was not found
 	##
-	## id: `Input::ID` enum value identifying the stream
+	## id: string value identifying the stream
-	global force_update: function(id: Input::ID) : bool;
+	global force_update: function(id: string) : bool;
-
-	## Adds a table filter to a specific input stream. Returns true on success.
-	##
-	## id: `Input::ID` enum value identifying the stream
-	## filter: the `TableFilter` record describing the filter.
-	global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool;
-
-	## Removes a named table filter to a specific input stream. Returns true on success.
-	##
-	## id: `Input::ID` enum value identifying the stream
-	## name: the name of the filter to be removed.
-	global remove_tablefilter: function(id: Input::ID, name: string) : bool;
-
-	## Adds an event filter to a specific input stream. Returns true on success.
-	##
-	## id: `Input::ID` enum value identifying the stream
-	## filter: the `EventFilter` record describing the filter.
-	global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool;
-
-	## Removes a named event filter to a specific input stream. Returns true on success.
-	##
-	## id: `Input::ID` enum value identifying the stream
-	## name: the name of the filter to be removed.
-	global remove_eventfilter: function(id: Input::ID, name: string) : bool;
-	#global get_filter: function(id: ID, name: string) : Filter;
-	
-	## Convenience function for reading a specific input source exactly once using 
-	## exactly one tablefilter
-	##
-	## id: `Input::ID` enum value identifying the stream
-	## description: `StreamDescription` record describing the source.
-	## filter: the `TableFilter` record describing the filter.	
-	global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool;
-
-	global update_finished: event(id: Input::ID);
 
+	## Event that is called, when the update of a specific source is finished
+	global update_finished: event(name: string, source:string);
 }
 
 @load base/input.bif
@@ -132,72 +118,26 @@ export {
 
 module Input;
 
-#global filters: table[ID, string] of Filter;
+#global streams: table[string] of Filter;
+# ^ change to set containing the names
 
-function create_stream(id: Input::ID, description: Input::StreamDescription) : bool
+function add_table(description: Input::TableDescription) : bool
 	{
-	return __create_stream(id, description);
+	return __create_table_stream(description);
 	}
 
-function remove_stream(id: Input::ID) : bool
+function add_event(description: Input::EventDescription) : bool
+	{
+	return __create_event_stream(description);
+	}
+
+function remove(id: string) : bool
 	{
 	return __remove_stream(id);
 	}
 
-function force_update(id: Input::ID) : bool
+function force_update(id: string) : bool
 	{
 	return __force_update(id);
 	}
 
-function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool
-	{
-#	filters[id, filter$name] = filter;
-	return __add_tablefilter(id, filter);
-	}
-
-function remove_tablefilter(id: Input::ID, name: string) : bool
-	{
-#	delete filters[id, name];
-	return __remove_tablefilter(id, name);
-	}
-
-function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool
-	{
-#	filters[id, filter$name] = filter;
-	return __add_eventfilter(id, filter);
-	}
-
-function remove_eventfilter(id: Input::ID, name: string) : bool
-	{
-#	delete filters[id, name];
-	return __remove_eventfilter(id, name);
-	}
-
-function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool {
-	local ok: bool = T;
-	# since we create and delete it ourselves this should be ok... at least for singlethreaded operation
-	local id: Input::ID = Input::TABLE_READ;
-
-	ok = create_stream(id, description);
-	if ( ok ) {
-		ok = add_tablefilter(id, filter);
-	}
-	if ( ok ) {
-		ok = force_update(id);
-	}
-	if ( ok ) {
-		ok = remove_stream(id);
-	} else {
-		remove_stream(id);
-	}
-
-	return ok;
-}
-
-#function get_filter(id: ID, name: string) : Filter
-#	{
-#	if ( [id, name] in filters )
-#		return filters[id, name];
-#
-#	return no_filter;
-#	}

src/DebugLogger.cc

@@ -15,7 +15,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
-	{ "logging", 0, false }, { "threading", 0, false }
+	{ "logging", 0, false }, {"input", 0, false }, 
+	{ "threading", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -24,6 +24,7 @@ enum DebugStream {
 	DBG_DPD,	// Dynamic application detection framework
 	DBG_TM,		// Time-machine packet input via Brocolli
 	DBG_LOGGING,	// Logging streams
+	DBG_INPUT,	// Input streams
 	DBG_THREADING,	// Threading system
 
 	NUM_DBGS // Has to be last

src/RemoteSerializer.cc

@@ -234,7 +234,7 @@ static const int PRINT_BUFFER_SIZE = 10 * 1024;
 static const int SOCKBUF_SIZE = 1024 * 1024;
 
 // Buffer size for remote-log data.
-static const int LOG_BUFFER_SIZE = 50 * 1024;
+static const int LOG_BUFFER_SIZE = 512;
 
 struct ping_args {
 	uint32 seq;
@@ -532,6 +532,7 @@ RemoteSerializer::RemoteSerializer()
 	terminating = false;
 	in_sync = 0;
 	last_flush = 0;
+	received_logs = 0;
 	}
 
 RemoteSerializer::~RemoteSerializer()
@@ -1353,6 +1354,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
 	{
 	Poll(false);
 
+	if ( received_logs > 0 )
+		{
+		// If we processed logs last time, assume there's more.
+		idle = false;
+		received_logs = 0;
+		return timer_mgr->Time();
+		}
+
 	double et = events.length() ? events[0]->time : -1;
 	double pt = packets.length() ? packets[0]->time : -1;
 
@@ -2552,7 +2561,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st
 	if ( ! peer->logs_requested )
 		return false;
 
-	assert(peer->log_buffer);
+	if ( ! peer->log_buffer )
+		// Peer shutting down.
+		return false;
 
 	// Serialize the log record entry.
 
@@ -2587,7 +2598,10 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st
 	if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) )
 		{
 		if ( ! FlushLogBuffer(peer) )
+			{
+			delete [] data;
 			return false;
+			}
 		}
 
 	// If the data is actually larger than our complete buffer, just send it out.
@@ -2631,6 +2645,12 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 	if ( current_peer->state == Peer::CLOSING )
 		return false;
 
+#ifdef USE_PERFTOOLS
+	// Don't track allocations here, they'll be released only after the
+	// main loop exists. And it's just a tiny amount anyway.
+	HeapLeakChecker::Disabler disabler;
+#endif
+
 	assert(current_args);
 
 	EnumVal* id_val = 0;
@@ -2666,7 +2686,7 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 	id_val = new EnumVal(id, BifType::Enum::Log::ID);
 	writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
 
-	if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) )
+	if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) )
 		goto error;
 
 	Unref(id_val);
@@ -2735,6 +2755,8 @@ bool RemoteSerializer::ProcessLogWrite()
 
 	fmt.EndRead();
 
+	++received_logs;
+
 	return true;
 
 error:
@@ -3376,6 +3398,9 @@ void SocketComm::Run()
 		small_timeout.tv_usec =
 			io->CanWrite() || io->CanRead() ? 1 : 10;
 
+		if ( ! io->CanWrite() )
+			usleep(10);
+
 		int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except,
 				&small_timeout);
 

src/RemoteSerializer.h

@@ -338,6 +338,7 @@ private:
 	int propagate_accesses;
 	bool ignore_accesses;
 	bool terminating;
+	int received_logs;
 	Peer* source_peer;
 	PeerID id_counter;	// Keeps track of assigned IDs.
 	uint32 current_sync_point;

src/input.bif

@@ -7,52 +7,33 @@ module Input;
 #include "NetVar.h"
 %%}
 
-type StreamDescription: record;
+type TableDescription: record;
-type TableFilter: record;
+type EventDescription: record;
-type EventFilter: record;
 
-function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool
+function Input::__create_table_stream%(description: Input::TableDescription%) : bool
 	%{
-	input::ReaderFrontend *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal());
+	bool res = input_mgr->CreateTableStream(description->AsRecordVal());
-	return new Val( the_reader != 0, TYPE_BOOL );
-	%}
-
-function Input::__remove_stream%(id: Input::ID%) : bool
-	%{
-	bool res = input_mgr->RemoveStream(id->AsEnumVal());
 	return new Val( res, TYPE_BOOL );
 	%}
 
-function Input::__force_update%(id: Input::ID%) : bool
+function Input::__create_event_stream%(description: Input::EventDescription%) : bool
 	%{
-	bool res = input_mgr->ForceUpdate(id->AsEnumVal());
+	bool res = input_mgr->CreateEventStream(description->AsRecordVal());
 	return new Val( res, TYPE_BOOL );
 	%}
 
-function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool
+function Input::__remove_stream%(id: string%) : bool
 	%{
-	bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal());
+	bool res = input_mgr->RemoveStream(id->AsString()->CheckString());
 	return new Val( res, TYPE_BOOL );
 	%}
 
-function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool
+function Input::__force_update%(id: string%) : bool
 	%{
-	bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString());
+	bool res = input_mgr->ForceUpdate(id->AsString()->CheckString());
-	return new Val( res, TYPE_BOOL);
-	%}
-
-function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool
-	%{
-	bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal());
 	return new Val( res, TYPE_BOOL );
 	%}
 
-function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool
-	%{
-	bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString());
-	return new Val( res, TYPE_BOOL);
-	%}
-
 # Options for Ascii Reader
 
 module InputAscii;

src/input/Manager.h

@@ -11,7 +11,7 @@
 #include "../EventHandler.h"
 #include "../RemoteSerializer.h"
 
-#include <vector>
+#include <map>
 
 namespace input {
 
@@ -35,6 +35,9 @@ public:
     
 	/**
 	 * Creates a new input stream.
+	 * Add a filter to an input source, which will write the data from the data source into 
+	 * a Bro table.
+	 * Add a filter to an input source, which sends events for read input data.
 	 *
 	 * @param id  The enum value corresponding the input stream.
 	 *
@@ -43,7 +46,9 @@ public:
 	 * This method corresponds directly to the internal BiF defined in
 	 * input.bif, which just forwards here.
 	 */	
-    	ReaderFrontend* CreateStream(EnumVal* id, RecordVal* description);
+    	bool CreateTableStream(RecordVal* description);
+    	bool CreateEventStream(RecordVal* description);
+
 
 	/**
 	 * Force update on a input stream.
@@ -57,7 +62,7 @@ public:
 	 * This method corresponds directly to the internal BiF defined in
 	 * input.bif, which just forwards here.
 	 */
-	bool ForceUpdate(const EnumVal* id);
+	bool ForceUpdate(const string &id);
 	
 	/**
 	 * Deletes an existing input stream
@@ -67,52 +72,7 @@ public:
 	 * This method corresponds directly to the internal BiF defined in
 	 * input.bif, which just forwards here.
 	 */
-	bool RemoveStream(const EnumVal* id);	
+	bool RemoveStream(const string &id);	
-
-	/** 
-	 * Add a filter to an input source, which will write the data from the data source into 
-	 * a Bro table.
-	 *
-	 * @param id  The enum value corresponding the input stream.
-	 * 
-	 * @param description A record of script type \c Input:TableFilter.
-	 *
-	 * This method corresponds directly to the internal BiF defined in
-	 * input.bif, which just forwards here.
-	 */
-	bool AddTableFilter(EnumVal *id, RecordVal* filter);
-
-	/**
-	 * Removes a tablefilter from the log stream
-	 *
-	 * @param id  The enum value corresponding the input stream.
-	 *
-	 * This method corresponds directly to the internal BiF defined in
-	 * input.bif, which just forwards here.
-	 */
-	bool RemoveTableFilter(EnumVal* id, const string &name);
-
-	/** 
-	 * Add a filter to an input source, which sends events for read input data.
-	 *
-	 * @param id  The enum value corresponding the input stream.
-	 * 
-	 * @param description A record of script type \c Input:EventFilter.
-	 *
-	 * This method corresponds directly to the internal BiF defined in
-	 * input.bif, which just forwards here.
-	 */
-	bool AddEventFilter(EnumVal *id, RecordVal* filter);
-
-	/**
-	 * Removes a eventfilter from the log stream
-	 *
-	 * @param id  The enum value corresponding the input stream.
-	 *
-	 * This method corresponds directly to the internal BiF defined in
-	 * input.bif, which just forwards here.
-	 */
-	bool RemoveEventFilter(EnumVal* id, const string &name);
 
 protected:
 	friend class ReaderFrontend;
@@ -122,19 +82,18 @@ protected:
 	friend class SendEventMessage;
 	friend class SendEntryMessage;
 	friend class EndCurrentSendMessage;
-	friend class FilterRemovedMessage;
 	friend class ReaderFinishedMessage;
 
 	// For readers to write to input stream in direct mode (reporting new/deleted values directly)
 	// Functions take ownership of threading::Value fields
-	void Put(const ReaderFrontend* reader, int id, threading::Value* *vals);
+	void Put(ReaderFrontend* reader, threading::Value* *vals);
-	void Clear(const ReaderFrontend* reader, int id);
+	void Clear(ReaderFrontend* reader);
-	bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals);
+	bool Delete(ReaderFrontend* reader, threading::Value* *vals);
 
 	// for readers to write to input stream in indirect mode (manager is monitoring new/deleted values)
 	// Functions take ownership of threading::Value fields
-	void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals);
+	void SendEntry(ReaderFrontend* reader, threading::Value* *vals);
-	void EndCurrentSend(const ReaderFrontend* reader, const int id);
+	void EndCurrentSend(ReaderFrontend* reader);
 	
 	// Allows readers to directly send Bro events.
 	// The num_vals and vals must be the same the named event expects.
@@ -150,20 +109,23 @@ protected:
 	// Used to prevent race conditions where data for a specific filter is still in the queue when the 
 	// RemoveFilter directive is executed by the main thread.
 	// This makes sure all data that has ben queued for a filter is still received.
-	bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId);
+	bool RemoveStreamContinuation(ReaderFrontend* reader);
-	bool RemoveStreamContinuation(const ReaderFrontend* reader);
 	
 private:
-	struct ReaderInfo;
+	class Filter;
+	class TableFilter;
+	class EventFilter;
+	
+    	bool CreateStream(Filter*, RecordVal* description);
 
 	// SendEntry implementation for Tablefilter
-	int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals);	
+	int SendEntryTable(Filter* i, const threading::Value* const *vals);	
 
 	// Put implementation for Tablefilter
-	int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals);	
+	int PutTable(Filter* i, const threading::Value* const *vals);	
 
 	// SendEntry and Put implementation for Eventfilter
-	int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals);
+	int SendEventFilterEvent(Filter* i, EnumVal* type, const threading::Value* const *vals);
 
 	// Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types 
 	// from the log framework
@@ -177,6 +139,9 @@ private:
 	void SendEvent(EventHandlerPtr ev, const int numvals, ...);	
 	void SendEvent(EventHandlerPtr ev, list<Val*> events);	
 
+	// Call predicate function and return result
+	bool CallPred(Func* pred_func, const int numvals, ...);
+
 	// get a hashkey for a set of threading::Values
 	HashKey* HashValues(const int num_elements, const threading::Value* const *vals);
 
@@ -197,16 +162,12 @@ private:
 	// Converts a Bro ListVal to a RecordVal given the record type
 	RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position);
 
-	ReaderInfo* FindReader(const ReaderFrontend* reader);
+	Filter* FindFilter(const string &name);
-	ReaderInfo* FindReader(const EnumVal* id);
+	Filter* FindFilter(ReaderFrontend* reader);
-
-	vector<ReaderInfo*> readers;
-
-	class Filter;
-	class TableFilter;
-	class EventFilter;
 
 	enum FilterType { TABLE_FILTER, EVENT_FILTER };
+	
+	map<ReaderFrontend*, Filter*> readers;
 };
 
 

src/input/ReaderBackend.cc

@@ -11,48 +11,44 @@ namespace input {
 
 class PutMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	PutMessage(ReaderFrontend* reader, int id, Value* *val)
+	PutMessage(ReaderFrontend* reader, Value* *val)
 		: threading::OutputMessage<ReaderFrontend>("Put", reader),
-		id(id), val(val) {}
+		val(val) {}
 
 	virtual bool Process() {
-		input_mgr->Put(Object(), id, val);
+		input_mgr->Put(Object(), val);
 		return true;
 	}
 
 private:
-	int id;
 	Value* *val;
 };
 
 class DeleteMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	DeleteMessage(ReaderFrontend* reader, int id, Value* *val)
+	DeleteMessage(ReaderFrontend* reader, Value* *val)
 		: threading::OutputMessage<ReaderFrontend>("Delete", reader),
-		id(id), val(val) {}
+		val(val) {}
 
 	virtual bool Process() {
-		return input_mgr->Delete(Object(), id, val);
+		return input_mgr->Delete(Object(), val);
 	}
 
 private:
-	int id;
 	Value* *val;
 };
 
 class ClearMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	ClearMessage(ReaderFrontend* reader, int id)
+	ClearMessage(ReaderFrontend* reader)
-		: threading::OutputMessage<ReaderFrontend>("Clear", reader),
+		: threading::OutputMessage<ReaderFrontend>("Clear", reader) {}
-		id(id) {}
 
 	virtual bool Process() {
-		input_mgr->Clear(Object(), id);
+		input_mgr->Clear(Object());
 		return true;
 	}
 
 private:
-	int id;
 };
 
 class SendEventMessage : public threading::OutputMessage<ReaderFrontend> {
@@ -73,47 +69,30 @@ private:
 
 class SendEntryMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	SendEntryMessage(ReaderFrontend* reader, const int id, Value* *val)
+	SendEntryMessage(ReaderFrontend* reader, Value* *val)
 		: threading::OutputMessage<ReaderFrontend>("SendEntry", reader),
-		id(id), val(val) { }
+		val(val) { }
 
 	virtual bool Process() {
-		input_mgr->SendEntry(Object(), id, val);
+		input_mgr->SendEntry(Object(), val);
 		return true;
 	}
 
 private:
-	const int id;
 	Value* *val;
 };
 
 class EndCurrentSendMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	EndCurrentSendMessage(ReaderFrontend* reader, const int id)
+	EndCurrentSendMessage(ReaderFrontend* reader)
-		: threading::OutputMessage<ReaderFrontend>("EndCurrentSend", reader),
+		: threading::OutputMessage<ReaderFrontend>("EndCurrentSend", reader) {}
-		id(id) {}
 
 	virtual bool Process() {
-		input_mgr->EndCurrentSend(Object(), id);
+		input_mgr->EndCurrentSend(Object());
 		return true;
 	}
 
 private:
-	const int id;
-};
-
-class FilterRemovedMessage : public threading::OutputMessage<ReaderFrontend> {
-public:
-	FilterRemovedMessage(ReaderFrontend* reader, const int id)
-		: threading::OutputMessage<ReaderFrontend>("FilterRemoved", reader),
-		id(id) {}
-
-	virtual bool Process() {
-		return input_mgr->RemoveFilterContinuation(Object(), id);
-	}
-
-private:
-	const int id;
 };
 
 class ReaderFinishedMessage : public threading::OutputMessage<ReaderFrontend> {
@@ -155,19 +134,19 @@ ReaderBackend::~ReaderBackend()
 	
 }
 
-void ReaderBackend::Put(int id, Value* *val) 
+void ReaderBackend::Put(Value* *val) 
 {
-	SendOut(new PutMessage(frontend, id, val));
+	SendOut(new PutMessage(frontend, val));
 }
 
-void ReaderBackend::Delete(int id, Value* *val) 
+void ReaderBackend::Delete(Value* *val) 
 {
-	SendOut(new DeleteMessage(frontend, id, val));
+	SendOut(new DeleteMessage(frontend, val));
 }
 
-void ReaderBackend::Clear(int id) 
+void ReaderBackend::Clear() 
 {
-	SendOut(new ClearMessage(frontend, id));
+	SendOut(new ClearMessage(frontend));
 }
 
 void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) 
@@ -175,70 +154,34 @@ void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *va
 	SendOut(new SendEventMessage(frontend, name, num_vals, vals));
 } 
 
-void ReaderBackend::EndCurrentSend(int id) 
+void ReaderBackend::EndCurrentSend() 
 {
-	SendOut(new EndCurrentSendMessage(frontend, id));
+	SendOut(new EndCurrentSendMessage(frontend));
 }
 
-void ReaderBackend::SendEntry(int id, Value* *vals)
+void ReaderBackend::SendEntry(Value* *vals)
 {
-	SendOut(new SendEntryMessage(frontend, id, vals));
+	SendOut(new SendEntryMessage(frontend, vals));
 }
 
-bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) 
+bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* arg_fields) 
 {
 	source = arg_source;
-	autostart = arg_autostart;
 	SetName("InputReader/"+source);
 
 	// disable if DoInit returns error.
-	disabled = !DoInit(arg_source, mode);
+	int success = DoInit(arg_source, mode, arg_num_fields, arg_fields);
 
-	if ( disabled ) {
+	if ( !success ) {
 		Error("Init failed");
 		DisableFrontend();
 	}
 
-	return !disabled;
+	disabled = !success;
-}
-
-bool ReaderBackend::StartReading() {
-	if ( disabled ) 
-		return false;
-
-	int success = DoStartReading();
-	
-	if ( success == false ) {
-		DisableFrontend();
-	} 
 
 	return success;
 }
 
-bool ReaderBackend::AddFilter(int id, int arg_num_fields,
-					   const Field* const * arg_fields) 
-{
-	if ( disabled ) 
-		return false;
-
-	bool success = DoAddFilter(id, arg_num_fields, arg_fields);
-	if ( success && autostart) {
-		autostart = false;
-		return StartReading();
-	}
-	return success;
-}
-
-bool ReaderBackend::RemoveFilter(int id) 
-{
-	if ( disabled ) 
-		return false;
-
-	bool success = DoRemoveFilter(id);
-	SendOut(new FilterRemovedMessage(frontend, id));
-	return success; // yes, I know, noone reads this.
-}
-
 void ReaderBackend::Finish() 
 {
 	DoFinish();

src/input/ReaderBackend.h

@@ -53,45 +53,13 @@ public:
 	 * 
 	 * @param mode the opening mode for the input source
 	 *
-	 * @param autostart automatically start the input source after the first filter has been added
-	 *
-	 * @return False if an error occured.
-	 */
-	bool Init(string arg_source, int mode, bool autostart);
-
-	/**
-	 * One-time start method of the reader.
-	 *
-	 * This method is called from the scripting layer, after all filters have been added.
-	 * No data should be read before this method is called.
-	 *
-	 * If autostart in Init is set to true, this method is called automatically by the backend after
-	 * the first filter has been added.
-	 */
-	bool StartReading();
-
-	/**
-	 * Add an input filter to the input stream
-	 *
-	 * @param id identifier of the input stream
-	 *
 	 * @param arg_num_fields number of fields contained in \a fields
 	 *
 	 * @param fields the types and names of the fields to be retrieved from the input source
 	 *
 	 * @return False if an error occured.
 	 */
-	bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields );
+	bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields);
-
-	
-	/**
-	 * Remove an input filter to the input stream
-	 *
-	 * @param id identifier of the input stream
-	 *
-	 * @return False if an error occured.
-	 */
-	bool RemoveFilter ( int id );
 
 	/**
 	 * Finishes reading from this input stream in a regular fashion. Must not be
@@ -131,33 +99,7 @@ protected:
 	 * disabled and eventually deleted. When returning false, an
 	 * implementation should also call Error() to indicate what happened.
 	 */
-	virtual bool DoInit(string arg_sources, int mode) = 0;
+	virtual bool DoInit(string arg_sources, int mode, int arg_num_fields, const threading::Field* const* fields) = 0;
-
-	/**
-	 * Reader-specific start method. After this function has been called, data may be read from
-	 * the input source and be sent to the specified filters
-	 *
-	 * A reader implementation must override this method.
-	 * If it returns false, it will be assumed that a fatal error has occured
-	 * that prevents the reader from further operation; it will then be
-	 * disabled and eventually deleted. When returning false, an implementation
-	 * should also call Error to indicate what happened.
-	 */
-	virtual bool DoStartReading() = 0;
-
-	/**
-	 * Reader-specific method to add a filter.
-	 *
-	 * A reader implementation must override this method.
-	 */
-	virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0;
-
-	/**
-	 * Reader-specific method to remove a filter.
-	 *
-	 * A reader implementation must override this method.
-	 */
-	virtual bool DoRemoveFilter( int id ) = 0;
 
 	/**
 	 * Reader-specific method implementing input finalization at
@@ -209,31 +151,26 @@ protected:
 	 *
 	 * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised
 	 *
-	 * @param id the input filter id for which the values are sent
-	 *
 	 * @param val list of threading::Values expected by the filter
 	 */
-	void Put(int id, threading::Value* *val);
+	void Put(threading::Value* *val);
 
 	/**
 	 * Method allowing a reader to delete a specific value from a bro table.
 	 *
 	 * If the receiving filter is an event, only a removed event is raised
 	 *
-	 * @param id the input filter id for which the values are sent
-	 *
 	 * @param val list of threading::Values expected by the filter
 	 */
-	void Delete(int id, threading::Value* *val);
+	void Delete(threading::Value* *val);
 
 	/**
 	 * Method allowing a reader to clear a value from a bro table.
 	 *
 	 * If the receiving filter is an event, this is ignored.
 	 *
-	 * @param id the input filter id for which the values are sent
 	 */
-	void Clear(int id);
+	void Clear();
 
 	// Content-sending-functions (tracking mode): Only changed lines are propagated.
 	
@@ -243,11 +180,9 @@ protected:
 	 *
 	 * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised.
 	 *
-	 * @param id the input filter id for which the values are sent
-	 *
 	 * @param val list of threading::Values expected by the filter
 	 */
-	void SendEntry(int id, threading::Value*  *vals);
+	void SendEntry(threading::Value*  *vals);
 
 	/**
 	 * Method telling the manager, that the current list of entries sent by SendEntry is finished.
@@ -255,9 +190,8 @@ protected:
 	 * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer
 	 * present in the input source
 	 *
-	 * @param id the input filter id for which the values are sent
 	 */
-	void EndCurrentSend(int id);
+	void EndCurrentSend();
 
 	/**
 	 * Triggered by regular heartbeat messages from the main thread.

src/input/ReaderFrontend.cc

@@ -12,16 +12,17 @@ namespace input {
 class InitMessage : public threading::InputMessage<ReaderBackend>
 {
 public:
-	InitMessage(ReaderBackend* backend, const string source, const int mode, const bool autostart)
+	InitMessage(ReaderBackend* backend, const string source, const int mode, const int num_fields, const threading::Field* const* fields)
 		: threading::InputMessage<ReaderBackend>("Init", backend),
-		source(source), mode(mode), autostart(autostart) { }
+		source(source), mode(mode), num_fields(num_fields), fields(fields) { }
 
-	virtual bool Process() { return Object()->Init(source, mode, autostart); }
+	virtual bool Process() { return Object()->Init(source, mode, num_fields, fields); }
 
 private:
 	const string source;
 	const int mode;
-	const bool autostart;
+	const int num_fields;
+       	const threading::Field* const* fields;
 };
 
 class UpdateMessage : public threading::InputMessage<ReaderBackend>
@@ -44,44 +45,6 @@ public:
 	virtual bool Process() { Object()->Finish(); return true; }
 };
 
-class StartReadingMessage : public threading::InputMessage<ReaderBackend>
-{
-public:
-	StartReadingMessage(ReaderBackend* backend)
-		: threading::InputMessage<ReaderBackend>("StartReading", backend)
-		 { }
-
-	virtual bool Process() { Object()->StartReading(); return true; }
-};
-
-class AddFilterMessage : public threading::InputMessage<ReaderBackend>
-{
-public:
-	AddFilterMessage(ReaderBackend* backend, const int id, const int num_fields, const threading::Field* const* fields)
-		: threading::InputMessage<ReaderBackend>("AddFilter", backend),
-		id(id), num_fields(num_fields), fields(fields) { }
-
-	virtual bool Process() { return Object()->AddFilter(id, num_fields, fields); }
-
-private:
-	const int id;
-	const int num_fields;
-       	const threading::Field* const* fields;
-};
-
-class RemoveFilterMessage : public threading::InputMessage<ReaderBackend>
-{
-public:
-	RemoveFilterMessage(ReaderBackend* backend, const int id)
-		: threading::InputMessage<ReaderBackend>("RemoveFilter", backend),
-		id(id) { }
-
-	virtual bool Process() { return Object()->RemoveFilter(id); }
-
-private:
-	const int id;
-};
-
 
 ReaderFrontend::ReaderFrontend(bro_int_t type) {
 	disabled = initialized = false;
@@ -95,7 +58,7 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) {
 ReaderFrontend::~ReaderFrontend() {
 }
 
-void ReaderFrontend::Init(string arg_source, int mode, bool autostart) {
+void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, const threading::Field* const* fields) {
 	if ( disabled )
 		return;
 
@@ -105,13 +68,18 @@ void ReaderFrontend::Init(string arg_source, int mode, bool autostart) {
 	source = arg_source;
 	initialized = true;
 
-	backend->SendIn(new InitMessage(backend, arg_source, mode, autostart));
+	backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields));
 } 
 
 void ReaderFrontend::Update() {
 	if ( disabled ) 
 		return;
 
+	if ( !initialized ) {
+		reporter->Error("Tried to call update on uninitialized reader");
+		return;
+	}
+
 	backend->SendIn(new UpdateMessage(backend));
 }
 
@@ -119,23 +87,14 @@ void ReaderFrontend::Finish() {
 	if ( disabled ) 
 		return;
 	
+	if ( !initialized ) {
+		reporter->Error("Tried to call finish on uninitialized reader");
+		return;
+	}
+
 	backend->SendIn(new FinishMessage(backend));
 }
 
-void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const threading::Field* const* fields) {
-	if ( disabled ) 
-		return;
-
-	backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields));
-}
-
-void ReaderFrontend::RemoveFilter(const int id) {
-	if ( disabled ) 
-		return;
-
-	backend->SendIn(new RemoveFilterMessage(backend, id));
-}
-
 string ReaderFrontend::Name() const
 {
 	if ( source.size() )
@@ -144,13 +103,5 @@ string ReaderFrontend::Name() const
 	return ty_name + "/" + source;
 }
 
-void ReaderFrontend::StartReading() {
-	if ( disabled ) 
-		return;
-
-	backend->SendIn(new StartReadingMessage(backend));
 }
 
-}
-
-

src/input/ReaderFrontend.h

@@ -49,18 +49,7 @@ public:
 	 * See ReaderBackend::Init() for arguments.
 	 * This method must only be called from the main thread.
 	 */	
-	void Init(string arg_source, int mode, bool autostart);
+	void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields);
-
-	/**
-	 * Start the reader.
-	 *
-	 * This methods starts the reader, after all necessary filters have been added.
-	 * It is not necessary to call this function, if autostart has been set.
-	 * If autostart has been set, the reader will be initialized automatically after the first filter has been added
-	 *
-	 * This method must only be called from the main thread.
-	 */
-	void StartReading();
 
 	/**
 	 * Force an update of the current input source. Actual action depends on
@@ -72,20 +61,6 @@ public:
 	 */
 	void Update();
 
-	/**
-	 * Add a filter to the current input source.
-	 *
-	 * See ReaderBackend::AddFilter for arguments.
-	 *
-	 * The method takes ownership of \a fields
-	 */
-	void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields );
-
-	/**
-	 * Removes a filter to the current input source.
-	 */
-	void RemoveFilter ( const int id );
-
 	/**
 	 * Finalizes writing to this tream.
 	 *

src/input/fdstream.h

@@ -0,0 +1,189 @@
+/* The following code declares classes to read from and write to
+ * file descriptore or file handles.
+ *
+ * See
+ *      http://www.josuttis.com/cppcode
+ * for details and the latest version.
+ *
+ * - open:
+ *      - integrating BUFSIZ on some systems?
+ *      - optimized reading of multiple characters
+ *      - stream for reading AND writing
+ *      - i18n
+ *
+ * (C) Copyright Nicolai M. Josuttis 2001.
+ * Permission to copy, use, modify, sell and distribute this software
+ * is granted provided this copyright notice appears in all copies.
+ * This software is provided "as is" without express or implied
+ * warranty, and with no claim as to its suitability for any purpose.
+ *
+ * Version: Jul 28, 2002
+ * History:
+ *  Jul 28, 2002: bugfix memcpy() => memmove()
+ *                fdinbuf::underflow(): cast for return statements
+ *  Aug 05, 2001: first public version
+ */
+#ifndef BOOST_FDSTREAM_HPP
+#define BOOST_FDSTREAM_HPP
+
+#include <istream>
+#include <ostream>
+#include <streambuf>
+// for EOF:
+#include <cstdio>
+// for memmove():
+#include <cstring>
+
+
+
+// low-level read and write functions
+#ifdef _MSC_VER
+# include <io.h>
+#else
+# include <sys/errno.h>
+# include <unistd.h>
+//extern "C" {
+//    int write (int fd, const char* buf, int num);
+//    int read (int fd, char* buf, int num);
+//}
+#endif
+
+
+// BEGIN namespace BOOST
+namespace boost {
+
+
+/************************************************************
+ * fdostream
+ * - a stream that writes on a file descriptor
+ ************************************************************/
+
+
+class fdoutbuf : public std::streambuf {
+  protected:
+    int fd;    // file descriptor
+  public:
+    // constructor
+    fdoutbuf (int _fd) : fd(_fd) {
+    }
+  protected:
+    // write one character
+    virtual int_type overflow (int_type c) {
+        if (c != EOF) {
+            char z = c;
+            if (write (fd, &z, 1) != 1) {
+                return EOF;
+            }
+        }
+        return c;
+    }
+    // write multiple characters
+    virtual
+    std::streamsize xsputn (const char* s,
+                            std::streamsize num) {
+        return write(fd,s,num);
+    }
+};
+
+class fdostream : public std::ostream {
+  protected:
+    fdoutbuf buf;
+  public:
+    fdostream (int fd) : std::ostream(0), buf(fd) {
+        rdbuf(&buf);
+    }
+};
+
+
+/************************************************************
+ * fdistream
+ * - a stream that reads on a file descriptor
+ ************************************************************/
+
+class fdinbuf : public std::streambuf {
+  protected:
+    int fd;    // file descriptor
+  protected:
+    /* data buffer:
+     * - at most, pbSize characters in putback area plus
+     * - at most, bufSize characters in ordinary read buffer
+     */
+    static const int pbSize = 4;        // size of putback area
+    static const int bufSize = 1024;    // size of the data buffer
+    char buffer[bufSize+pbSize];        // data buffer
+
+  public:
+    /* constructor
+     * - initialize file descriptor
+     * - initialize empty data buffer
+     * - no putback area
+     * => force underflow()
+     */
+    fdinbuf (int _fd) : fd(_fd) {
+        setg (buffer+pbSize,     // beginning of putback area
+              buffer+pbSize,     // read position
+              buffer+pbSize);    // end position
+    }
+
+  protected:
+    // insert new characters into the buffer
+    virtual int_type underflow () {
+#ifndef _MSC_VER
+        using std::memmove;
+#endif
+
+        // is read position before end of buffer?
+        if (gptr() < egptr()) {
+            return traits_type::to_int_type(*gptr());
+        }
+
+        /* process size of putback area
+         * - use number of characters read
+         * - but at most size of putback area
+         */
+        int numPutback;
+        numPutback = gptr() - eback();
+        if (numPutback > pbSize) {
+            numPutback = pbSize;
+        }
+
+        /* copy up to pbSize characters previously read into
+         * the putback area
+         */
+        memmove (buffer+(pbSize-numPutback), gptr()-numPutback,
+                numPutback);
+
+        // read at most bufSize new characters
+        int num;
+        num = read (fd, buffer+pbSize, bufSize);
+	if ( num == EAGAIN ) {
+		return 0;
+	}
+        if (num <= 0) {
+            // ERROR or EOF
+            return EOF;
+        }
+
+        // reset buffer pointers
+        setg (buffer+(pbSize-numPutback),   // beginning of putback area
+              buffer+pbSize,                // read position
+              buffer+pbSize+num);           // end of buffer
+
+        // return next character
+        return traits_type::to_int_type(*gptr());
+    }
+};
+
+class fdistream : public std::istream {
+  protected:
+    fdinbuf buf;
+  public:
+    fdistream (int fd) : std::istream(0), buf(fd) {
+        rdbuf(&buf);
+    }
+};
+
+
+} // END namespace boost
+
+#endif /*BOOST_FDSTREAM_HPP*/

src/input/readers/Ascii.cc

@@ -26,6 +26,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int
 {
 	position = arg_position;
 	secondary_position = -1;
+	present = true;
 }
 
 FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) 
@@ -33,10 +34,11 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, cons
 {
 	position = arg_position;
 	secondary_position = -1;
+	present = true;
 }
 
 FieldMapping::FieldMapping(const FieldMapping& arg) 
-	: name(arg.name), type(arg.type), subtype(arg.subtype)
+	: name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present)
 {
 	position = arg.position;
 	secondary_position = arg.secondary_position;
@@ -76,7 +78,6 @@ Ascii::~Ascii()
 
 void Ascii::DoFinish()
 {
-	filters.empty();
 	if ( file != 0 ) {
 		file->close();
 		delete(file);
@@ -84,13 +85,15 @@ void Ascii::DoFinish()
 	}
 }
 
-bool Ascii::DoInit(string path, int arg_mode)
+bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields)
 {
-	started = false;
 	fname = path;
 	mode = arg_mode;
 	mtime = 0;
 	
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
 	if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) {
 		Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str()));
 		return false;
@@ -108,16 +111,6 @@ bool Ascii::DoInit(string path, int arg_mode)
 		return false;
 	}
 	
-	return true;
-}
-
-bool Ascii::DoStartReading() {
-	if ( started == true ) {
-		Error("Started twice");
-		return false;
-	}	
-
-	started = true;
 	switch ( mode ) {
 		case MANUAL:
 		case REREAD:
@@ -131,46 +124,11 @@ bool Ascii::DoStartReading() {
 	return true;
 }
 
-bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) {
-	if ( HasFilter(id) ) {
-		Error("Filter was added twice, ignoring.");		
-		return false; // no, we don't want to add this a second time
-	}
-
-	Filter f;
-	f.num_fields = arg_num_fields;
-	f.fields = fields;
-
-	filters[id] = f;
-
-	return true;
-}
-
-bool Ascii::DoRemoveFilter ( int id ) {
-	if (!HasFilter(id) ) {
-		Error("Filter removal of nonexisting filter requested.");		
-		return false;
-	}
-
-	assert ( filters.erase(id) == 1 );
-
-	return true;
-}	
-
-
-bool Ascii::HasFilter(int id) {
-	map<int, Filter>::iterator it = filters.find(id);	
-	if ( it == filters.end() ) {
-		return false;
-	}
-	return true;
-}
-
 
 bool Ascii::ReadHeader(bool useCached) {
 	// try to read the header line...
 	string line;
-	map<string, uint32_t> fields;
+	map<string, uint32_t> ifields;
 
 	if ( !useCached ) {
 		if ( !GetLine(line) ) {
@@ -194,38 +152,44 @@ bool Ascii::ReadHeader(bool useCached) {
 		if ( !getline(splitstream, s, separator[0]))
 			break;
 
-		fields[s] = pos;
+		ifields[s] = pos;
 		pos++;
 	}
 
 	//printf("Updating fields from description %s\n", line.c_str());
-	for ( map<int, Filter>::iterator it = filters.begin(); it != filters.end(); it++ ) {
+	columnMap.clear();
-		(*it).second.columnMap.clear();
 		
-		for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) {
+	for ( unsigned int i = 0; i < num_fields; i++ ) {
-			const Field* field = (*it).second.fields[i];
+		const Field* field = fields[i];
 		
-			map<string, uint32_t>::iterator fit = fields.find(field->name);	
+		map<string, uint32_t>::iterator fit = ifields.find(field->name);	
-			if ( fit == fields.end() ) {
+		if ( fit == ifields.end() ) {
-				Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str()));
+			if ( field->optional ) {
-				return false;
+				// we do not really need this field. mark it as not present and always send an undef back.
+				FieldMapping f(field->name, field->type, field->subtype, -1);
+				f.present = false;
+				columnMap.push_back(f);
+				continue;
 			}
 
-
+			Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str()));
-			FieldMapping f(field->name, field->type, field->subtype, fields[field->name]);
+			return false;
-			if ( field->secondary_name != "" ) {
-				map<string, uint32_t>::iterator fit2 = fields.find(field->secondary_name);					
-				if ( fit2 == fields.end() ) {
-					Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str()));
-					return false;
-				}
-				f.secondary_position = fields[field->secondary_name];
-			}
-			(*it).second.columnMap.push_back(f);
 		}
 
+
+		FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]);
+		if ( field->secondary_name != "" ) {
+			map<string, uint32_t>::iterator fit2 = ifields.find(field->secondary_name);					
+			if ( fit2 == ifields.end() ) {
+				Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str()));
+				return false;
+			}
+			f.secondary_position = ifields[field->secondary_name];
+		}
+		columnMap.push_back(f);
 	}
 
+	
 	// well, that seems to have worked...
 	return true;
 }
@@ -266,7 +230,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) {
 		} else if ( s == "F" ) {
 			val->val.int_val = 0;
 		} else {
-			Error(Fmt("Invalid value for boolean: %s", s.c_str()));
+			Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str()));
 			return false;
 		}
 		break;
@@ -461,57 +425,63 @@ bool Ascii::DoUpdate() {
 
 		pos--; // for easy comparisons of max element.
 
-		for ( map<int, Filter>::iterator it = filters.begin(); it != filters.end(); it++ ) {
 		
-			Value** fields = new Value*[(*it).second.num_fields];
+		Value** fields = new Value*[num_fields];
 
-			int fpos = 0;
+		int fpos = 0;
-			for ( vector<FieldMapping>::iterator fit = (*it).second.columnMap.begin();
+		for ( vector<FieldMapping>::iterator fit = columnMap.begin();
-				fit != (*it).second.columnMap.end();
+			fit != columnMap.end();
-				fit++ ){
+			fit++ ){
-
-
-				if ( (*fit).position > pos || (*fit).secondary_position > pos ) {
-					Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos,  (*fit).position, (*fit).secondary_position));
-					return false;
-				}
-
-				Value* val = EntryToVal(stringfields[(*fit).position], *fit);
-				if ( val == 0 ) {
-					Error("Could not convert String value to Val");
-					return false;
-				}
-				
-				if ( (*fit).secondary_position != -1 ) {
-					// we have a port definition :)
-					assert(val->type == TYPE_PORT ); 
-					//	Error(Fmt("Got type %d != PORT with secondary position!", val->type));
-
-					val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]);
-				}
-
-				fields[fpos] = val;
 
+			if ( ! fit->present ) {
+				// add non-present field
+				fields[fpos] =  new Value((*fit).type, false);
 				fpos++;
+				continue;
 			}
 			
-			//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
+			assert(fit->position >= 0 );
-			assert ( (unsigned int) fpos == (*it).second.num_fields );
 
-			if ( mode == STREAM ) {
+			if ( (*fit).position > pos || (*fit).secondary_position > pos ) {
-				Put((*it).first, fields);
+				Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos,  (*fit).position, (*fit).secondary_position));
-			} else {
+				return false;
-				SendEntry((*it).first, fields);
 			}
 
-			/* Do not do this, ownership changes to other thread 
+			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
-			 * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) {
+			if ( val == 0 ) {
-				delete fields[i];
+				Error("Could not convert String value to Val");
+				return false;
 			}
-			delete [] fields;
+			
-			*/
+			if ( (*fit).secondary_position != -1 ) {
+				// we have a port definition :)
+				assert(val->type == TYPE_PORT ); 
+				//	Error(Fmt("Got type %d != PORT with secondary position!", val->type));
+
+				val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]);
+			}
+
+			fields[fpos] = val;
+
+			fpos++;
 		}
 
+		//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
+		assert ( (unsigned int) fpos == num_fields );
+
+		if ( mode == STREAM ) {
+			Put(fields);
+		} else {
+			SendEntry(fields);
+		}
+
+		/* Do not do this, ownership changes to other thread 
+		 * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) {
+			delete fields[i];
+		}
+		delete [] fields;
+		*/
+
 	}
 
 
@@ -519,9 +489,7 @@ bool Ascii::DoUpdate() {
 	//file->seekg(0, ios::beg); // and seek to start.
 
 	if ( mode != STREAM ) {
-		for ( map<int, Filter>::iterator it = filters.begin(); it != filters.end(); it++ ) {
+		EndCurrentSend();
-			EndCurrentSend((*it).first);
-		}
 	}
 
 	return true;

src/input/readers/Ascii.h

@@ -19,6 +19,7 @@ struct FieldMapping {
 	int position;
 	// for ports: pos of the second field
 	int secondary_position;
+	bool present;
 
 	FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); 
 	FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); 
@@ -39,33 +40,22 @@ public:
     
 protected:
 	
-	virtual bool DoInit(string path, int mode);
+	virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields);
-
-	virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields );
-
-	virtual bool DoRemoveFilter ( int id );	
 
 	virtual void DoFinish();
 
 	virtual bool DoUpdate();
 
-	virtual bool DoStartReading();
-    
 private:
 
 	virtual bool DoHeartbeat(double network_time, double current_time);
 
-	struct Filter {
+	unsigned int num_fields;
-		unsigned int num_fields;
 
-		const threading::Field* const * fields; // raw mapping		
+	const threading::Field* const * fields; // raw mapping		
 
-		// map columns in the file to columns to send back to the manager
+	// map columns in the file to columns to send back to the manager
-		vector<FieldMapping> columnMap;		
+	vector<FieldMapping> columnMap;		
-
-	};
-
-	bool HasFilter(int id);
 
 	bool ReadHeader(bool useCached);
 	threading::Value* EntryToVal(string s, FieldMapping type);
@@ -75,8 +65,6 @@ private:
 	ifstream* file;
 	string fname;
 
-	map<int, Filter> filters;
-
 	// Options set from the script-level.
 	string separator;
 
@@ -91,7 +79,6 @@ private:
 
 	int mode;
 
-	bool started;
 	time_t mtime;
 
 };

src/input/readers/Raw.cc

@@ -7,6 +7,7 @@
 #include <sstream>
 
 #include "../../threading/SerialTypes.h"
+#include "../fdstream.h"
 
 #define MANUAL 0
 #define REREAD 1
@@ -15,6 +16,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <stdio.h>
 
 using namespace input::reader;
 using threading::Value;
@@ -23,6 +25,7 @@ using threading::Field;
 Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend)
 {
 	file = 0;
+	in = 0;
 
 	//keyMap = new map<string, string>();
 	
@@ -40,57 +43,74 @@ Raw::~Raw()
 
 void Raw::DoFinish()
 {
-	filters.empty();
 	if ( file != 0 ) {
-		file->close();
+		Close();
-		delete(file);
-		file = 0;
 	}
 }
 
-bool Raw::DoInit(string path, int arg_mode)
+bool Raw::Open() 
+{
+	if ( execute ) {
+		file = popen(fname.c_str(), "r");
+		if ( file == NULL ) {
+			Error(Fmt("Could not execute command %s", fname.c_str()));
+			return false;
+		}
+	} else {
+		file = fopen(fname.c_str(), "r");
+		if ( file == NULL ) {
+			Error(Fmt("Init: cannot open %s", fname.c_str()));
+			return false;
+		}
+	}
+	
+	in = new boost::fdistream(fileno(file));
+
+	if ( execute && mode == STREAM ) {
+		fcntl(fileno(file), F_SETFL, O_NONBLOCK);
+	}
+
+	return true;
+}
+
+bool Raw::Close()
+{
+	if ( file == NULL ) {
+		InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str()));
+		return false;
+	}
+
+	if ( execute ) {
+		delete(in);
+		pclose(file);
+	} else {
+		delete(in);
+		fclose(file);
+	}
+
+	in = NULL;
+	file = NULL;
+
+	return true;
+}
+
+bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields)
 {
-	started = false;
 	fname = path;
 	mode = arg_mode;
 	mtime = 0;
+	execute = false;
+	firstrun = true;
+	bool result;
 
-	if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) {
+	num_fields = arg_num_fields;
-		Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str()));
+	fields = arg_fields;
+
+	if ( path.length() == 0 ) {
+		Error("No source path provided");
 		return false;
 	}
 	
-	file = new ifstream(path.c_str());
-	if ( !file->is_open() ) {
-		Error(Fmt("Init: cannot open %s", fname.c_str()));
-		return false;
-	}
-
-	return true;
-}
-
-bool Raw::DoStartReading() {
-	if ( started == true ) {
-		Error("Started twice");
-		return false;
-	}	
-
-	started = true;
-	switch ( mode ) {
-		case MANUAL:
-		case REREAD:
-		case STREAM:
-			DoUpdate();
-			break;
-		default:
-			assert(false);
-	}
-
-	return true;
-}
-
-bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) {
-
 	if ( arg_num_fields != 1 ) {
 		Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored.");
 		return false;
@@ -101,42 +121,51 @@ bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields )
 		return false;
 	}
 
-	if ( HasFilter(id) ) {
+	// do Initialization
-		Error("Filter was added twice, ignoring");
+	char last = path[path.length()-1];
-		return false; // no, we don't want to add this a second time
+	if ( last == '|' ) {
+		execute = true;
+		fname = path.substr(0, fname.length() - 1);
+
+		if ( ( mode != MANUAL ) && ( mode != STREAM ) ) {
+			Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str()));
+			return false;
+		} 	
+		
+		result = Open();
+
+	} else {
+		execute = false;
+		if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) {
+			Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str()));
+			return false;
+		} 
+
+		result = Open();	
+
 	}
 
-	Filter f;
+	if ( result == false ) {
-	f.num_fields = arg_num_fields;
+		return result;
-	f.fields = fields;
+	}
 
-	filters[id] = f;
 
+#ifdef DEBUG
+	Debug(DBG_INPUT, "Raw reader created, will perform first update");
+#endif
+
+	// after initialization - do update
+	DoUpdate();
+
+#ifdef DEBUG
+	Debug(DBG_INPUT, "First update went through");
+#endif
 	return true;
 }
 
-bool Raw::DoRemoveFilter ( int id ) {
-	if (!HasFilter(id) ) {
-		Error("Filter removal of nonexisting filter requested.");				
-		return false;
-	}
-
-	assert ( filters.erase(id) == 1 );
-
-	return true;
-}	
-
-
-bool Raw::HasFilter(int id) {
-	map<int, Filter>::iterator it = filters.find(id);	
-	if ( it == filters.end() ) {
-		return false;
-	}
-	return true;
-}
 
 bool Raw::GetLine(string& str) {
-	while ( getline(*file, str, separator[0]) ) {
+	while ( getline(*in, str, separator[0]) ) {
 		return true;
 	}
 
@@ -146,63 +175,59 @@ bool Raw::GetLine(string& str) {
 
 // read the entire file and send appropriate thingies back to InputMgr
 bool Raw::DoUpdate() {
-	switch ( mode ) {
+	if ( firstrun ) {
-		case REREAD:
+		firstrun = false;
-			// check if the file has changed
+	} else {
-			struct stat sb;
+		switch ( mode ) {
-			if ( stat(fname.c_str(), &sb) == -1 ) {
+			case REREAD:
-				Error(Fmt("Could not get stat for %s", fname.c_str()));
+				// check if the file has changed
-				return false;
+				struct stat sb;
-			}
+				if ( stat(fname.c_str(), &sb) == -1 ) {
+					Error(Fmt("Could not get stat for %s", fname.c_str()));
+					return false;
+				}
 
-			if ( sb.st_mtime <= mtime ) {
+				if ( sb.st_mtime <= mtime ) {
-				// no change
+					// no change
-				return true;
+					return true;
-			}
+				}
 
-			mtime = sb.st_mtime;
+				mtime = sb.st_mtime;
-			// file changed. reread.
+				// file changed. reread.
 
-			// fallthrough
+				// fallthrough
-		case MANUAL:
+			case MANUAL:
-		case STREAM:
+			case STREAM:
-
+				Debug(DBG_INPUT, "Updating");
-			if ( file && file->is_open() ) {
+				if ( mode == STREAM && file != NULL && in != NULL ) {
-				if ( mode == STREAM ) {
+					fpurge(file);
-					file->clear(); // remove end of file evil bits
+					in->clear(); // remove end of file evil bits
 					break;
 				}
-				file->close();
-			}
-			file = new ifstream(fname.c_str());
-			if ( !file->is_open() ) {
-				Error(Fmt("cannot open %s", fname.c_str()));
-				return false;
-			}
 
-			break;
+				Close();
-		default:
+				if ( !Open() ) {
-			assert(false);
+					return false;
+				}
+				break;
+			default:
+				assert(false);
 
+		}
 	}
 
 	string line;
 	while ( GetLine(line) ) {
-		for ( map<int, Filter>::iterator it = filters.begin(); it != filters.end(); it++ ) {
+		assert (num_fields == 1);
 	
-			assert ((*it).second.num_fields == 1);
+		Value** fields = new Value*[1];
 
-			Value** fields = new Value*[1];
+		// filter has exactly one text field. convert to it.
-
+		Value* val = new Value(TYPE_STRING, true);
-			// filter has exactly one text field. convert to it.
+		val->val.string_val = new string(line);
-			Value* val = new Value(TYPE_STRING, true);
+		fields[0] = val;
-			val->val.string_val = new string(line);
-			fields[0] = val;
-			
-			Put((*it).first, fields);
-
-		}
 		
+		Put(fields);
 	}
 
 	return true;
@@ -213,6 +238,8 @@ bool Raw::DoHeartbeat(double network_time, double current_time)
 {
 	ReaderBackend::DoHeartbeat(network_time, current_time);
 
+	Debug(DBG_INPUT, "Heartbeat");
+
 	switch ( mode ) {
 		case MANUAL:
 			// yay, we do nothing :)

src/input/readers/Raw.h

@@ -19,36 +19,25 @@ public:
     
 protected:
 	
-	virtual bool DoInit(string path, int mode);
+	virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields);
-
-	virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields );
-
-	virtual bool DoRemoveFilter ( int id );	
 
 	virtual void DoFinish();
 
 	virtual bool DoUpdate();
 
-	virtual bool DoStartReading();
-    
 private:
 
 	virtual bool DoHeartbeat(double network_time, double current_time);
-
+	bool Open();
-	struct Filter {
+	bool Close();
-		unsigned int num_fields;
-
-		const threading::Field* const * fields; // raw mapping		
-	};
-
-	bool HasFilter(int id);
 
 	bool GetLine(string& str);
 	
-	ifstream* file;
+	istream* in;
-	string fname;
 
-	map<int, Filter> filters;
+	FILE* file;
+	
+	string fname;
 
 	// Options set from the script-level.
 	string separator;
@@ -57,10 +46,15 @@ private:
 	string headerline;
 
 	int mode;
+	bool execute;
+	bool firstrun;
 
-	bool started;
 	time_t mtime;
 	
+	unsigned int num_fields;
+
+	const threading::Field* const * fields; // raw mapping		
+
 };
 
 

src/logging/Manager.cc

@@ -105,9 +105,6 @@ Manager::Stream::~Stream()
 		{
 		WriterInfo* winfo = i->second;
 
-		if ( ! winfo )
-			continue;
-
 		if ( winfo->rotation_timer )
 			timer_mgr->Cancel(winfo->rotation_timer);
 
@@ -207,7 +204,7 @@ Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer)
 			{
 			WriterInfo* winfo = i->second;
 
-			if ( winfo && winfo->writer == writer )
+			if ( winfo->writer == writer )
 				return winfo;
 			}
 		}
@@ -221,7 +218,7 @@ void Manager::RemoveDisabledWriters(Stream* stream)
 
 	for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ )
 		{
-		if ( j->second && j->second->writer->Disabled() )
+		if ( j->second->writer->Disabled() )
 			{
 			j->second->writer->Stop();
 			delete j->second;
@@ -680,11 +677,11 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 			Val* path_arg;
 			if ( filter->path_val )
-				path_arg = filter->path_val;
+				path_arg = filter->path_val->Ref();
 			else
 				path_arg = new StringVal("");
 
-			vl.append(path_arg->Ref());
+			vl.append(path_arg);
 
 			Val* rec_arg;
 			BroType* rt = filter->path_func->FType()->Args()->FieldType("rec");
@@ -718,7 +715,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 			if ( ! filter->path_val )
 				{
-				Unref(path_arg);
 				filter->path = v->AsString()->CheckString();
 				filter->path_val = v->Ref();
 				}
@@ -740,7 +736,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 		if ( w != stream->writers.end() )
 			// We know this writer already.
-			writer = w->second ? w->second->writer : 0;
+			writer = w->second->writer;
 
 		else
 			{
@@ -753,64 +749,25 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 			for ( int j = 0; j < filter->num_fields; ++j )
 				arg_fields[j] = new Field(*filter->fields[j]);
 
-			if ( filter->remote )
+			writer = CreateWriter(stream->id, filter->writer,
-				remote_serializer->SendLogCreateWriter(stream->id,
+					      path, filter->num_fields,
-								       filter->writer,
+					      arg_fields, filter->local, filter->remote);
-								       path,
-								       filter->num_fields,
-								       arg_fields);
 
-			if ( filter->local )
+			if ( ! writer )
 				{
-				writer = CreateWriter(stream->id, filter->writer,
+				Unref(columns);
-						      path, filter->num_fields,
+				return false;
-						      arg_fields);
-
-				if ( ! writer )
-					{
-					Unref(columns);
-					return false;
-					}
 				}
-			else
-				{
-				// Insert a null pointer into the map to make
-				// sure we don't try creating it again.
-				stream->writers.insert(Stream::WriterMap::value_type(
-				Stream::WriterPathPair(filter->writer->AsEnum(), path), 0));
 
-				for( int i = 0; i < filter->num_fields; ++i)
-					delete arg_fields[i];
-
-				delete [] arg_fields;
-				}
 			}
 
 		// Alright, can do the write now.
 
-		if ( filter->local || filter->remote )
+		threading::Value** vals = RecordToFilterVals(stream, filter, columns);
-			{
-				threading::Value** vals = RecordToFilterVals(stream, filter, columns);
-
-			if ( filter->remote )
-				remote_serializer->SendLogWrite(stream->id,
-								filter->writer,
-								path,
-								filter->num_fields,
-								vals);
-
-			if ( filter->local )
-				{
-				// Write takes ownership of vals.
-				assert(writer);
-				writer->Write(filter->num_fields, vals);
-				}
-
-			else
-				DeleteVals(filter->num_fields, vals);
-
-			}
 
+		// Write takes ownership of vals.
+		assert(writer);
+		writer->Write(filter->num_fields, vals);
 
 #ifdef DEBUG
 		DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'",
@@ -976,7 +933,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 	}
 
 WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
-				int num_fields, const Field* const*  fields)
+				int num_fields, const Field* const*  fields, bool local, bool remote)
 	{
 	Stream* stream = FindStream(id);
 
@@ -987,12 +944,12 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path,
 	Stream::WriterMap::iterator w =
 		stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path));
 
-	if ( w != stream->writers.end() && w->second )
+	if ( w != stream->writers.end() )
 		// If we already have a writer for this. That's fine, we just
 		// return it.
 		return w->second->writer;
 
-	WriterFrontend* writer_obj = new WriterFrontend(writer->AsEnum());
+	WriterFrontend* writer_obj = new WriterFrontend(id, writer, local, remote);
 	assert(writer_obj);
 
 	writer_obj->Init(path, num_fields, fields);
@@ -1089,8 +1046,7 @@ bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields,
 		return false;
 		}
 
-	if ( w->second )
+	w->second->writer->Write(num_fields, vals);
-		w->second->writer->Write(num_fields, vals);
 
 	DBG_LOG(DBG_LOGGING,
 		"Wrote pre-filtered record to path '%s' on stream '%s'",
@@ -1111,9 +1067,6 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
 		for ( Stream::WriterMap::iterator i = stream->writers.begin();
 		      i != stream->writers.end(); i++ )
 			{
-			if ( ! i->second )
-				continue;
-
 			WriterFrontend* writer = i->second->writer;
 
 			EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
@@ -1134,10 +1087,7 @@ bool Manager::SetBuf(EnumVal* id, bool enabled)
 
 	for ( Stream::WriterMap::iterator i = stream->writers.begin();
 	      i != stream->writers.end(); i++ )
-		{
+		i->second->writer->SetBuf(enabled);
-		if ( i->second )
-			i->second->writer->SetBuf(enabled);
-		}
 
 	RemoveDisabledWriters(stream);
 
@@ -1155,10 +1105,7 @@ bool Manager::Flush(EnumVal* id)
 
 	for ( Stream::WriterMap::iterator i = stream->writers.begin();
 	      i != stream->writers.end(); i++ )
-		{
+		i->second->writer->Flush();
-		if ( i->second )
-			i->second->writer->Flush();
-		}
 
 	RemoveDisabledWriters(stream);
 

src/logging/Manager.h

@@ -159,7 +159,8 @@ protected:
 
 	// Takes ownership of fields.
 	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path,
-				int num_fields, const threading::Field* const* fields);
+				int num_fields, const threading::Field* const* fields,
+				bool local, bool remote);
 
 	// Takes ownership of values..
 	bool Write(EnumVal* id, EnumVal* writer, string path,

src/logging/WriterFrontend.cc

@@ -99,21 +99,36 @@ public:
 
 using namespace logging;
 
-WriterFrontend::WriterFrontend(bro_int_t type)
+WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
 	{
+	stream = arg_stream;
+	writer = arg_writer;
+	Ref(stream);
+	Ref(writer);
+
 	disabled = initialized = false;
 	buf = true;
+	local = arg_local;
+	remote = arg_remote;
 	write_buffer = 0;
 	write_buffer_pos = 0;
 	ty_name = "<not set>";
-	backend = log_mgr->CreateBackend(this, type);
 
-	assert(backend);
+	if ( local )
-	backend->Start();
+		{
+		backend = log_mgr->CreateBackend(this, writer->AsEnum());
+		assert(backend);
+		backend->Start();
+		}
+
+	else
+		backend = 0;
 	}
 
 WriterFrontend::~WriterFrontend()
 	{
+	Unref(stream);
+	Unref(writer);
 	}
 
 string WriterFrontend::Name() const
@@ -128,7 +143,9 @@ void WriterFrontend::Stop()
 	{
 	FlushWriteBuffer();
 	SetDisable();
-	backend->Stop();
+
+	if ( backend )
+		backend->Stop();
 	}
 
 void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields)
@@ -144,7 +161,17 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons
 	fields = arg_fields;
 
 	initialized = true;
-	backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields));
+
+	if ( backend )
+		backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields));
+
+	if ( remote )
+		remote_serializer->SendLogCreateWriter(stream,
+						       writer,
+						       arg_path,
+						       arg_num_fields,
+						       arg_fields);
+
 	}
 
 void WriterFrontend::Write(int num_fields, Value** vals)
@@ -152,6 +179,19 @@ void WriterFrontend::Write(int num_fields, Value** vals)
 	if ( disabled )
 		return;
 
+	if ( remote )
+		remote_serializer->SendLogWrite(stream,
+						writer,
+						path,
+						num_fields,
+						vals);
+
+	if ( ! backend )
+		{
+		DeleteVals(vals);
+		return;
+		}
+
 	if ( ! write_buffer )
 		{
 		// Need new buffer.
@@ -173,7 +213,8 @@ void WriterFrontend::FlushWriteBuffer()
 		// Nothing to do.
 		return;
 
-	backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer));
+	if ( backend )
+		backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer));
 
 	// Clear buffer (no delete, we pass ownership to child thread.)
 	write_buffer = 0;
@@ -187,7 +228,8 @@ void WriterFrontend::SetBuf(bool enabled)
 
 	buf = enabled;
 
-	backend->SendIn(new SetBufMessage(backend, enabled));
+	if ( backend )
+		backend->SendIn(new SetBufMessage(backend, enabled));
 
 	if ( ! buf )
 		// Make sure no longer buffer any still queued data.
@@ -200,7 +242,9 @@ void WriterFrontend::Flush()
 		return;
 
 	FlushWriteBuffer();
-	backend->SendIn(new FlushMessage(backend));
+
+	if ( backend )
+		backend->SendIn(new FlushMessage(backend));
 	}
 
 void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
@@ -209,7 +253,9 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool
 		return;
 
 	FlushWriteBuffer();
-	backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
+
+	if ( backend )
+		backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
 	}
 
 void WriterFrontend::Finish()
@@ -218,7 +264,18 @@ void WriterFrontend::Finish()
 		return;
 
 	FlushWriteBuffer();
-	backend->SendIn(new FinishMessage(backend));
+
+	if ( backend )
+		backend->SendIn(new FinishMessage(backend));
+	}
+
+void WriterFrontend::DeleteVals(Value** vals)
+	{
+	// Note this code is duplicated in Manager::DeleteVals().
+	for ( int i = 0; i < num_fields; i++ )
+		delete vals[i];
+
+	delete [] vals;
 	}
 
 

src/logging/WriterFrontend.h

@@ -25,14 +25,21 @@ public:
 	/**
 	 * Constructor.
 	 *
-	 * type: The backend writer type, with the value corresponding to the
+	 * stream: The logging stream.
+	 *
+	 * writer: The backend writer type, with the value corresponding to the
 	 * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The
 	 * frontend will internally instantiate a WriterBackend of the
 	 * corresponding type.
 	 * 
+	 * local: If true, the writer will instantiate a local backend.
+	 *
+	 * remote: If true, the writer will forward all data to remote
+	 * clients.
+	 *
 	 * Frontends must only be instantiated by the main thread.
 	 */
-	WriterFrontend(bro_int_t type);
+	WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote);
 
 	/**
 	 * Destructor.
@@ -187,10 +194,17 @@ public:
 protected:
 	friend class Manager;
 
+	void DeleteVals(threading::Value** vals);
+
+	EnumVal* stream;
+	EnumVal* writer;
+
 	WriterBackend* backend;	// The backend we have instanatiated.
 	bool disabled;	// True if disabled.
 	bool initialized;	// True if initialized.
 	bool buf;	// True if buffering is enabled (default).
+	bool local;	// True if logging locally.
+	bool remote;	// True if loggin remotely.
 
 	string ty_name;	// Name of the backend type. Set by the manager.
 	string path;	// The log path.

src/threading/BasicThread.cc

@@ -20,8 +20,8 @@ BasicThread::BasicThread()
 	terminating = false;
 	pthread = 0;
 
-	buf = 0;
+	buf_len = 2048;
-	buf_len = 1024;
+	buf = (char*) malloc(buf_len);
 
 	name = Fmt("thread-%d", ++thread_counter);
 
@@ -57,9 +57,6 @@ void BasicThread::SetOSName(const string& name)
 
 const char* BasicThread::Fmt(const char* format, ...)
 	{
-	if ( ! buf )
-		buf = (char*) malloc(buf_len);
-
 	va_list al;
 	va_start(al, format);
 	int n = safe_vsnprintf(buf, buf_len, format, al);
@@ -67,13 +64,15 @@ const char* BasicThread::Fmt(const char* format, ...)
 
 	if ( (unsigned int) n >= buf_len )
 		{ // Not enough room, grow the buffer.
-		buf_len = n + 32;
+		int tmp_len = n + 32;
-		buf = (char*) realloc(buf, buf_len);
+		char* tmp = (char*) malloc(tmp_len);
 
 		// Is it portable to restart?
 		va_start(al, format);
-		n = safe_vsnprintf(buf, buf_len, format, al);
+		n = safe_vsnprintf(tmp, tmp_len, format, al);
 		va_end(al);
+
+		free(tmp);
 		}
 
 	return buf;

src/threading/Manager.cc

@@ -102,25 +102,25 @@ void Manager::Process()
 			next_beat = 0;
 			}
 
-		if ( ! t->HasOut() )
+		while ( t->HasOut() )
-			continue;
-
-		Message* msg = t->RetrieveOut();
-
-		if ( msg->Process() ) 
 			{
-			//if ( network_time ) //&& network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init.
+			Message* msg = t->RetrieveOut();
 
-				did_process = true;
+			if ( msg->Process() )
-			}
+				{
-		else
+				//if ( network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init.
-			{
+					did_process = true;
-			string s = msg->Name() + " failed, terminating thread " + t->Name() + " (in ThreadManager)";
+				}
-			reporter->Error("%s", s.c_str());
+
-			t->Stop();
+			else
+				{
+				string s = msg->Name() + " failed, terminating thread";
+				reporter->Error("%s", s.c_str());
+				t->Stop();
 			}
 
-		delete msg;
+			delete msg;
+			}
 		}
 
 //	fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat);

src/threading/MsgThread.cc

@@ -281,7 +281,7 @@ void MsgThread::GetStats(Stats* stats)
 	{
 	stats->sent_in = cnt_sent_in;
 	stats->sent_out = cnt_sent_out;
-	stats->pending_in = cnt_sent_in - queue_in.Size();
+	stats->pending_in = queue_in.Size();
-	stats->pending_out = cnt_sent_out - queue_out.Size();
+	stats->pending_out = queue_out.Size();
 	}
 

src/threading/SerialTypes.cc

@@ -13,7 +13,7 @@ bool Field::Read(SerializationFormat* fmt)
 	int st;
 
 	bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && 
-			fmt->Read(&t, "type") && fmt->Read(&st, "subtype") );
+			fmt->Read(&t, "type") && fmt->Read(&st, "subtype") && fmt->Read(&optional, "optional"));
 	type = (TypeTag) t;
 	subtype = (TypeTag) st;
 
@@ -23,7 +23,7 @@ bool Field::Read(SerializationFormat* fmt)
 bool Field::Write(SerializationFormat* fmt) const
 	{
 	return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && 
-			fmt->Write((int)subtype, "subtype"));
+			fmt->Write((int)subtype, "subtype"), fmt->Write(optional, "optional"));
 	}
 
 Value::~Value()

src/threading/SerialTypes.h

@@ -24,17 +24,18 @@ struct Field {
 	string secondary_name;	
 	TypeTag type;	//! Type of the field.
 	TypeTag subtype;	//! Inner type for sets.
+	bool optional; //! needed by input framework. Is the field optional or does it have to be present in the input data
 
 	/**
 	 * Constructor.
 	 */
-	Field() 	{ subtype = TYPE_VOID; }
+	Field() 	{ subtype = TYPE_VOID; optional = false; }
 
 	/**
 	 * Copy constructor.
 	 */
 	Field(const Field& other)
-		: name(other.name), type(other.type), subtype(other.subtype) {  }
+		: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) {  }
 
 	/**
 	 * Unserializes a field.

src/types.bif

@@ -182,10 +182,6 @@ enum Event %{
 	EVENT_REMOVED,
 %}
 
-enum ID %{
-	Unknown,
-%}
-
 enum Mode %{
 	MANUAL = 0,
 	REREAD = 1,

testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	metrics
+#fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
+#types	time	enum	string	addr	string	subnet	count
+1331256494.591966	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1331256494.591966	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1331256494.591966	TEST_METRIC	foo-bar	1.2.3.4	-	-	6

testing/btest/Baseline/core.leaks.remote/sender.test.failure.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test.failure
+#fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+#types	time	addr	port	addr	port	string	string
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/core.leaks.remote/sender.test.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+#types	time	addr	port	addr	port	string	string
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/core.leaks.remote/sender.test.success.log

@@ -0,0 +1,9 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test.success
+#fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+#types	time	addr	port	addr	port	string	string
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR

testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out

@@ -0,0 +1,4 @@
+{
+[2] = [b=<uninitialized>],
+[1] = [b=T]
+}

testing/btest/Baseline/scripts.base.frameworks.input.event/out

@@ -1,21 +1,70 @@
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 1
 T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 2
 T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 3
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 4
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 5
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 6
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::i;
+print A::b;
+}]
 Input::EVENT_NEW
 7
 T

testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out

@@ -0,0 +1,145 @@
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+dfsdf
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdf
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.
+[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+A::try = A::try + 1;
+if (9 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+
+done

testing/btest/Baseline/scripts.base.frameworks.input.optional/out

@@ -0,0 +1,9 @@
+{
+[2] = [b=T, notb=F],
+[4] = [b=F, notb=T],
+[6] = [b=F, notb=T],
+[7] = [b=T, notb=F],
+[1] = [b=T, notb=F],
+[5] = [b=F, notb=T],
+[3] = [b=F, notb=T]
+}

testing/btest/Baseline/scripts.base.frameworks.input.raw/out

@@ -1,8 +1,64 @@
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 q3r3057fdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 sdfs\d
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 dfsdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 sdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
 3rw43wRRERLlL#RWERERERE.

testing/btest/Baseline/scripts.base.frameworks.input.repeat/out

@@ -0,0 +1,160 @@
+input0
+input.log
+{
+[1] = T
+}
+input1
+input.log
+{
+[1] = T
+}
+input2
+input.log
+{
+[1] = T
+}
+input3
+input.log
+{
+[1] = T
+}
+input4
+input.log
+{
+[1] = T
+}
+input5
+input.log
+{
+[1] = T
+}
+input6
+input.log
+{
+[1] = T
+}
+input7
+input.log
+{
+[1] = T
+}
+input8
+input.log
+{
+[1] = T
+}
+input9
+input.log
+{
+[1] = T
+}
+input10
+input.log
+{
+[1] = T
+}
+input11
+input.log
+{
+[1] = T
+}
+input12
+input.log
+{
+[1] = T
+}
+input13
+input.log
+{
+[1] = T
+}
+input14
+input.log
+{
+[1] = T
+}
+input15
+input.log
+{
+[1] = T
+}
+input16
+input.log
+{
+[1] = T
+}
+input17
+input.log
+{
+[1] = T
+}
+input18
+input.log
+{
+[1] = T
+}
+input19
+input.log
+{
+[1] = T
+}
+input20
+input.log
+{
+[1] = T
+}
+input21
+input.log
+{
+[1] = T
+}
+input22
+input.log
+{
+[1] = T
+}
+input23
+input.log
+{
+[1] = T
+}
+input24
+input.log
+{
+[1] = T
+}
+input25
+input.log
+{
+[1] = T
+}
+input26
+input.log
+{
+[1] = T
+}
+input27
+input.log
+{
+[1] = T
+}
+input28
+input.log
+{
+[1] = T
+}
+input29
+input.log
+{
+[1] = T
+}
+input30
+input.log
+{
+[1] = T
+}
+input31
+input.log
+{
+[1] = T
+}

testing/btest/Baseline/scripts.base.frameworks.input.reread/out

@@ -14,8 +14,44 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_NEW
+Left
 [i=-42]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -59,8 +95,56 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_NEW
+Left
 [i=-43]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -116,8 +200,56 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_CHANGED
+Left
 [i=-43]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -172,9 +304,177 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-============EVENT============
+============PREDICATE============
 Input::EVENT_NEW
+[i=-45]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_NEW
+[i=-46]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_NEW
+[i=-47]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_NEW
+[i=-48]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
 [i=-44]
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -187,9 +487,117 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_NEW
+Left
 [i=-45]
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -203,9 +611,81 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
-Input::EVENT_NEW
+Description
-[i=-45]
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
@@ -217,9 +697,30 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_NEW
+Left
 [i=-46]
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -233,9 +734,81 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
-Input::EVENT_NEW
+Description
-[i=-46]
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
@@ -247,9 +820,30 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_NEW
+Left
 [i=-47]
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -263,9 +857,81 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
-Input::EVENT_NEW
+Description
-[i=-47]
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
@@ -277,24 +943,30 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-Input::EVENT_NEW
+{ 
-[i=-48]
+print A::outfile, ============EVENT============;
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+print A::outfile, Description;
-2,
+print A::outfile, A::description;
-4,
+print A::outfile, Type;
-1,
+print A::outfile, A::tpe;
-3
+print A::outfile, Left;
-}, ss={
+print A::outfile, A::left;
-CC,
+print A::outfile, Right;
-AA,
+print A::outfile, A::right;
-BB
+}, pred=anonymous-function
-}, se={
+{ 
-
+print A::outfile, ============PREDICATE============;
-}, vc=[10, 20, 30], ve=[]]
+print A::outfile, A::typ;
-============EVENT============
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
 Input::EVENT_NEW
+Left
 [i=-48]
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -485,8 +1157,11 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
+Description
 Input::EVENT_REMOVED
+Type
 [i=-43]
+Left
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -499,9 +1174,13 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+Right
 ============EVENT============
+Description
 Input::EVENT_REMOVED
+Type
 [i=-46]
+Left
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -514,9 +1193,13 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+Right
 ============EVENT============
+Description
 Input::EVENT_REMOVED
+Type
 [i=-44]
+Left
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -529,9 +1212,13 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+Right
 ============EVENT============
+Description
 Input::EVENT_REMOVED
+Type
 [i=-47]
+Left
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -544,9 +1231,13 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+Right
 ============EVENT============
+Description
 Input::EVENT_REMOVED
+Type
 [i=-45]
+Left
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -559,9 +1250,13 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+Right
 ============EVENT============
+Description
 Input::EVENT_REMOVED
+Type
 [i=-42]
+Left
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -574,6 +1269,7 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+Right
 ==========SERVERS============
 {
 [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={

testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out

@@ -0,0 +1,128 @@
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+dfsdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+dfsdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+sdf
+[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::description;
+print A::tpe;
+print A::s;
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.

testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out

@@ -0,0 +1,120 @@
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+q3r3057fdf
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdfs\d
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+dfsdf
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+sdf
+[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=<no value description>, want_record=F, ev=line
+{ 
+print A::outfile, A::description;
+print A::outfile, A::tpe;
+print A::outfile, A::s;
+if (3 == A::try) 
+{ 
+print A::outfile, done;
+close(A::outfile);
+Input::remove(input);
+}
+
+}]
+Input::EVENT_NEW
+3rw43wRRERLlL#RWERERERE.

testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out

@@ -1,21 +1,126 @@
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=1]
 T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=2]
 T
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=3]
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=4]
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=5]
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=6]
 F
+[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={
+[2] = T,
+[4] = F,
+[6] = F,
+[7] = T,
+[1] = T,
+[5] = F,
+[3] = F
+}, idx=<no value description>, val=<no value description>, want_record=F, ev=line
+{ 
+print description;
+print tpe;
+print left;
+print right;
+}, pred=<uninitialized>]
 Input::EVENT_NEW
 [i=7]
 T

testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out

@@ -1,15 +0,0 @@
-VALID
-VALID
-VALID
-VALID
-VALID
-VALID
-VALID
-MARK
-VALID
-VALID
-VALID
-VALID
-VALID
-VALID
-VALID

testing/btest/Baseline/scripts.base.frameworks.input.twotables/out

@@ -0,0 +1,349 @@
+============PREDICATE============
+Input::EVENT_NEW
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE 2============
+Input::EVENT_NEW
+[i=-43]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=-42]
+Right
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+==========SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+============EVENT============
+Description
+[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh2, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE 2============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=-43]
+Right
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+==========SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
+============PREDICATE============
+Input::EVENT_NEW
+[i=-44]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_REMOVED
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=-44]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+Input::EVENT_REMOVED
+Type
+[i=-42]
+Left
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+Right
+==========SERVERS============
+{
+[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]],
+[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}

testing/btest/core/leaks/basic-cluster.bro

@@ -0,0 +1,39 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
+# @TEST-EXEC: btest-bg-wait -k 30
+# @TEST-EXEC: btest-diff manager-1/metrics.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+redef Log::default_rotation_interval = 0secs;
+
+redef enum Metrics::ID += {
+	TEST_METRIC,
+};
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter(TEST_METRIC, 
+		[$name="foo-bar",
+		 $break_interval=3secs]);
+		
+	if ( Cluster::local_node_type() == Cluster::WORKER )
+		{
+		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+		}
+	}

testing/btest/core/leaks/remote.bro

@@ -0,0 +1,79 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-diff sender/test.log
+# @TEST-EXEC: btest-diff sender/test.failure.log
+# @TEST-EXEC: btest-diff sender/test.success.log
+# @TEST-EXEC: cmp receiver/test.log sender/test.log
+# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
+# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+
+# This is the common part loaded by both sender and receiver.
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { LOG };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Log]);
+	Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]);
+}
+
+#####
+
+@TEST-START-FILE sender.bro
+
+module Test;
+
+@load frameworks/communication/listen
+
+function fail(rec: Log): bool
+	{
+	return rec$status != "success";
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]);
+
+	local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	local r: Log = [$t=network_time(), $id=cid, $status="success"];
+
+	# Log something.
+	Log::write(Test::LOG, r);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	disconnect(p);
+	}
+@TEST-END-FILE
+
+@TEST-START-FILE receiver.bro
+
+#####
+
+redef Communication::nodes += {
+    ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
+};
+
+@TEST-END-FILE

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -45,12 +41,10 @@ global servers: table[int] of Val = table();
 event bro_init()
 {
 	# first read in the old stuff into the table...
-	Input::create_stream(A::INPUT, [$source="input.log"]);
+	Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
-	Input::remove_tablefilter(A::INPUT, "ssh");
-	Input::remove_stream(A::INPUT);
 }
 
-event Input::update_finished(id: Input::ID) {
+event Input::update_finished(name: string, source:string) {
 	print servers;
 }

testing/btest/scripts/base/frameworks/input/emptyvals.bro

@@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i
+##types	bool	int
+T	1
+-	2
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+}
+
+event Input::update_finished(name: string, source:string) {
+	print servers;
+}

testing/btest/scripts/base/frameworks/input/event.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -19,16 +19,13 @@
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Val: record {
 	i: int;
 	b: bool;
 };
 
-event line(tpe: Input::Event, i: int, b: bool) {
+event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) {
+	print description;
 	print tpe;
 	print i;
 	print b;
@@ -36,6 +33,6 @@ event line(tpe: Input::Event, i: int, b: bool) {
 
 event bro_init()
 {
-	Input::create_stream(A::INPUT, [$source="input.log"]);
+	Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]);
-	Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
 }

testing/btest/scripts/base/frameworks/input/executeraw.bro

@@ -0,0 +1,33 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print description;
+	print tpe;
+	print s;
+}
+
+event bro_init()
+{
+	Input::add_event([$source="wc input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/executestreamraw.bro

@@ -0,0 +1,58 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	try = try + 1;
+	
+	if ( try == 9 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+}

testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -30,12 +26,11 @@ global servers: table[int] of Val = table();
 
 event bro_init()
 {
-	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]);
-	Input::create_stream(A::INPUT, [$source="input.log"]);
+	Input::remove("input");
-	Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]);
 }
 
-event Input::update_finished(id: Input::ID) {
+event Input::update_finished(name: string, source: string) {
 	print servers;
 }
 

testing/btest/scripts/base/frameworks/input/onecolumn-record.bro

@@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -30,12 +26,11 @@ global servers: table[int] of Val = table();
 
 event bro_init()
 {
-	# first read in the old stuff into the table...
+	Input::add_table([$name="input", $source="input.log", $idx=Idx, $val=Val, $destination=servers]);
-	Input::create_stream(A::INPUT, [$source="input.log"]);
+	Input::remove("input");
-	Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]);
 }
 
-event Input::update_finished(id: Input::ID) {
+event Input::update_finished(name: string, source: string) {
 	print servers;
 }
 

testing/btest/scripts/base/frameworks/input/optional.bro

@@ -0,0 +1,45 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	notb: bool &optional;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
+				$pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; }
+				]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers;
+}

testing/btest/scripts/base/frameworks/input/port.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -13,10 +13,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: addr;
 };
@@ -29,17 +25,14 @@ global servers: table[addr] of Val = table();
 
 event bro_init()
 {
-	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]);
-	Input::create_stream(A::INPUT, [$source="input.log"]);
-	Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]);
 	print servers[1.2.3.4];
 	print servers[1.2.3.5];
 	print servers[1.2.3.6];
-	Input::remove_tablefilter(A::INPUT, "input");
+	Input::remove("input");
-	Input::remove_stream(A::INPUT);
 }
 
-event Input::update_finished(id: Input::ID) {
+event Input::update_finished(name: string, source: string) {
 	print servers[1.2.3.4];
 	print servers[1.2.3.5];
 	print servers[1.2.3.6];

testing/btest/scripts/base/frameworks/input/predicate-stream.bro

@@ -23,10 +23,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -38,7 +34,7 @@ type Val: record {
 global servers: table[int] of Val = table();
 global ct: int;
 
-event line(tpe: Input::Event, left: Idx, right: bool) {
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) {
 	ct = ct + 1;
 	if ( ct < 3 ) {
 		return;
@@ -75,9 +71,10 @@ event bro_init()
 {
 	ct = 0;
 	# first read in the old stuff into the table...
-	Input::create_stream(A::INPUT, [$source="input.log", $mode=Input::STREAM]);
+	Input::add_table([$source="input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line,
-	Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line,
 				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
 				]);
+	Input::remove("input");
+
 }
 

testing/btest/scripts/base/frameworks/input/predicate.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -20,10 +20,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -37,13 +33,13 @@ global servers: table[int] of Val = table();
 event bro_init()
 {
 	# first read in the old stuff into the table...
-	Input::create_stream(A::INPUT, [$source="input.log"]);
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F,
-	Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F,
 				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
 				]);
+	Input::remove("input");
 }
 
-event Input::update_finished(id: Input::ID) {
+event Input::update_finished(name: string, source: string) {
 	if ( 1 in servers ) {
 		print "VALID";
 	}

testing/btest/scripts/base/frameworks/input/raw.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -16,20 +16,18 @@ sdf
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Val: record {
 	s: string;
 };
 
-event line(tpe: Input::Event, s: string) {
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print description;
+	print tpe;
 	print s;
 }
 
 event bro_init()
 {
-	Input::create_stream(A::INPUT, [$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM]);
+	Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
-	Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
 }

testing/btest/scripts/base/frameworks/input/repeat.bro

@@ -0,0 +1,41 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global destination: table[int] of Val = table();
+
+const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32};
+
+event bro_init()
+{
+	for ( i in one_to_32 ) {
+		Input::add_table([$source="input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]);
+		Input::remove(fmt("input%d", i));
+	}
+}
+
+event Input::update_finished(name: string, source: string) {
+	print name;
+	print source;
+	print destination;
+}

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -62,10 +62,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -94,10 +90,15 @@ global outfile: file;
 
 global try: count;
 
-event line(tpe: Input::Event, left: Idx, right: Val) {
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
 	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
 	print outfile, tpe;
+	print outfile, "Left";
 	print outfile, left;
+	print outfile, "Right";
 	print outfile, right;
 }
 
@@ -106,8 +107,7 @@ event bro_init()
 	outfile = open ("../out");
 	try = 0;
 	# first read in the old stuff into the table...
-	Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]);
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
-	Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
 	$pred(typ: Input::Event, left: Idx, right: Val) = { 
 	print outfile, "============PREDICATE============";
 	print outfile, typ;
@@ -119,7 +119,7 @@ event bro_init()
 }
 
 
-event Input::update_finished(id: Input::ID) {
+event Input::update_finished(name: string, source: string) {
 	print outfile, "==========SERVERS============";
 	print outfile, servers;
 	
@@ -127,7 +127,6 @@ event Input::update_finished(id: Input::ID) {
 	if ( try == 5 ) {
 		print outfile, "done";
 		close(outfile);
-		Input::remove_tablefilter(A::INPUT, "ssh");
+		Input::remove("input");
-		Input::remove_stream(A::INPUT);
 	}
 }

testing/btest/scripts/base/frameworks/input/rereadraw.bro

@@ -0,0 +1,34 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print description;
+	print tpe;
+	print s;
+}
+
+event bro_init()
+{
+	Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]);
+	Input::force_update("input");
+	Input::remove("input");
+}

testing/btest/scripts/base/frameworks/input/stream.bro

@@ -28,10 +28,6 @@ redef InputAscii::empty_field = "EMPTY";
 
 module A;
 
-export {
-	redef enum Input::ID += { INPUT };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -60,7 +56,7 @@ global outfile: file;
 
 global try: count;
 
-event line(tpe: Input::Event, left: Idx, right: Val) {
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
 	print outfile, "============EVENT============";
 	print outfile, tpe;
 	print outfile, left;
@@ -73,8 +69,7 @@ event line(tpe: Input::Event, left: Idx, right: Val) {
 	if ( try == 3 ) {
 		print outfile, "done";
 		close(outfile);
-		Input::remove_tablefilter(A::INPUT, "ssh");
+		Input::remove("input");
-		Input::remove_stream(A::INPUT);
 	}
 }
 
@@ -83,7 +78,6 @@ event bro_init()
 	outfile = open ("../out");
 	try = 0;
 	# first read in the old stuff into the table...
-	Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::STREAM]);
+	Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]);
-	Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]);
 }
 

testing/btest/scripts/base/frameworks/input/streamraw.bro

@@ -0,0 +1,56 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string) {
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	
+	if ( try == 3 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+}

testing/btest/scripts/base/frameworks/input/tableevent.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -18,12 +18,6 @@
 
 redef InputAscii::empty_field = "EMPTY";
 
-module A;
-
-export {
-	redef enum Log::ID += { LOG };
-}
-
 type Idx: record {
 	i: int;
 };
@@ -34,7 +28,8 @@ type Val: record {
 
 global destination: table[int] of Val = table();
 
-event line(tpe: Input::Event, left: Idx, right: bool) {
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) {
+	print description;
 	print tpe;
 	print left;
 	print right;
@@ -42,6 +37,6 @@ event line(tpe: Input::Event, left: Idx, right: bool) {
 
 event bro_init()
 {
-	Input::create_stream(A::LOG, [$source="input.log"]);
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]);
-	Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]);
+	Input::remove("input");
 }

testing/btest/scripts/base/frameworks/input/twofilters.bro

@@ -1,104 +0,0 @@
-#
-# @TEST-EXEC: bro %INPUT >out
-# @TEST-EXEC: btest-diff out
-
-@TEST-START-FILE input.log
-#separator \x09
-#path	ssh
-#fields	i	b	
-#types	int	bool
-1	T
-2	T
-3	F
-4	F
-5	F
-6	F
-7	T
-@TEST-END-FILE
-
-redef InputAscii::empty_field = "EMPTY";
-
-module A;
-
-export {
-	redef enum Input::ID += { INPUT };
-}
-
-type Idx: record {
-	i: int;
-};
-
-type Val: record {
-	b: bool;
-};
-
-global destination1: table[int] of Val = table();
-global destination2: table[int] of Val = table();
-
-global done: bool = F;
-
-event bro_init()
-{
-	# first read in the old stuff into the table...
-	Input::create_stream(A::INPUT, [$source="input.log", $autostart=F]);
-	Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F,
-				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
-				]);
-	Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]);
-	
-	Input::force_update(A::INPUT);
-}
-
-event Input::update_finished(id: Input::ID) {
-        if ( done == T ) {
-                return;
-        }
-
-        done = T;
-
-	if ( 1 in destination1 ) {
-		print "VALID";
-	}
-	if ( 2 in destination1 ) {
-		print "VALID";
-	}
-	if ( !(3 in destination1) ) {
-		print "VALID";
-	}
-	if ( !(4 in destination1) ) {
-		print "VALID";
-	}
-	if ( !(5 in destination1) ) {
-		print "VALID";
-	}
-	if ( !(6 in destination1) ) {
-		print "VALID";
-	}
-	if ( 7 in destination1 ) {
-		print "VALID";
-	}
-
-	print "MARK";
-
-	if ( 2 in destination2 ) {
-		print "VALID";
-	}
-	if ( 2 in destination2 ) {
-		print "VALID";
-	}
-	if ( 3 in destination2 ) {
-		print "VALID";
-	}
-	if ( 4 in destination2 ) {
-		print "VALID";
-	}
-	if ( 5 in destination2 ) {
-		print "VALID";
-	}
-	if ( 6 in destination2 ) {
-		print "VALID";
-	}
-	if ( 7 in destination2 ) {
-		print "VALID";
-	}
-}

testing/btest/scripts/base/frameworks/input/twotables.bro

@@ -0,0 +1,113 @@
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro %INPUT 
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: btest-bg-wait -k 2
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+F	-44	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+}
+
+event bro_init()
+{
+	outfile = open ("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE 2============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+}
+
+
+event Input::update_finished(name: string, source: string) {
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 5 ) {
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+	}
+}

testing/external/scripts/perftools-adapt-paths

@@ -7,4 +7,4 @@
 
 cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1
 
-grep -q "No leaks found" $1
+grep -qv "detected leaks of" $1