Merge branch 'master' of ssh://git.bro-ids.org/bro

CHANGES

@@ -1,4 +1,49 @@
 
+2.1-1128 | 2013-08-24 10:27:29 -0700
+
+  * Remove code relict in input framework. (Jon Siwek)
+    
+  * Fix documentation for mkdir BIF. (Jon Siwek)
+    
+  * File extraction tweaks. (Jon Siwek)
+
+    - Default extraction limit of 100MB now provided via a tuning
+      script loaded in local.bro so that command-line Bro is unlimited
+      by default.
+
+    - Extraction directory is now created on request of file
+      extraction rather than unconditionally in bro_init(). (Jon
+      Siwek)
+
+2.1-1124 | 2013-08-23 16:33:52 -0700
+
+  * Fixed a number of object bugs DNP3 analyzer. (Hui Lin)
+    
+2.1-1122 | 2013-08-22 16:52:27 -0700
+
+  * Use macros to create file analyzer plugin classes. (Jon Siwek)
+    
+  * Add options to limit extracted file sizes w/ 100MB default. (Jon
+    Siwek)
+
+2.1-1117 | 2013-08-22 08:44:12 -0700
+
+  * A number of input framework fixes and corresponding test stability
+    improvements. (Jon Siwek)
+
+  * Make memory leak tests able to time out.  (Jon Siwek)
+    
+  * Fix a compiler warning regarding strncat misuse. (Jon Siwek)
+    
+2.1-1103 | 2013-08-21 19:11:34 -0400
+
+  * A number of sumstats fixes. (Seth Hall, Vlad Grigorescu)
+    
+  * Fix memory leak w/ when statements. Addresses BIT-1058. (Jon
+    Siwek)
+
+  * Switching to relative submodule paths (Robin Sommer)
+
 2.1-1089 | 2013-08-19 11:25:11 -0700
 
   * Fix bloom filters' dependence on size_t. (Jon Siwek, Matthias

VERSION

@@ -1 +1 @@
-2.1-1089
+2.1-1128

aux/btest

@@ -1 +1 @@
-Subproject commit be7c653dcdc30384d4d17359d19d94540fdedaa5
+Subproject commit 35bb074c1c5173e44689df680a24ba13fea39a11

doc/scripts/DocSourcesList.cmake

@@ -36,6 +36,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
@@ -264,6 +266,7 @@ rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
+rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
 rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)

scripts/base/files/extract/main.bro

@@ -7,6 +7,10 @@ export {
 	## The prefix where files are extracted to.
 	const prefix = "./extract_files/" &redef;
 
+	## The default max size for extracted files (they won't exceed this
+	## number of bytes), unlimited.
+	const default_limit = 0 &redef;
+
 	redef record Files::Info += {
 		## Local filenames of extracted file.
 		extracted: string &optional &log;
@@ -17,9 +21,32 @@ export {
 		## This field is used in the core by the extraction plugin
 		## to know where to write the file to.  It's also optional
 		extract_filename: string &optional;
+		## The maximum allowed file size in bytes of *extract_filename*.
+		## Once reached, a :bro:see:`file_extraction_limit` event is
+		## raised and the analyzer will be removed unless
+		## :bro:see:`FileExtract::set_limit` is called to increase the
+		## limit.  A value of zero means "no limit".
+		extract_limit: count &default=default_limit;
 	};
+
+	## Sets the maximum allowed extracted file size.
+	##
+	## f: A file that's being extracted.
+	##
+	## args: Arguments that identify a file extraction analyzer.
+	##
+	## n: Allowed number of bytes to be extracted.
+	##
+	## Returns: false if a file extraction analyzer wasn't active for
+	##          the file, else true.
+	global set_limit: function(f: fa_file, args: Files::AnalyzerArgs, n: count): bool;
 }
 
+function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
+	{
+	return __set_limit(f$id, args, n);
+	}
+
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
@@ -27,12 +54,10 @@ function on_add(f: fa_file, args: Files::AnalyzerArgs)
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);
+	mkdir(prefix);
 	}
 
 event bro_init() &priority=10
 	{
 	Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, on_add);
-
-	# Create the extraction directory.
-	mkdir(prefix);
 	}

scripts/policy/tuning/defaults/__load__.bro

@@ -1,2 +1,3 @@
 @load ./packet-fragments
 @load ./warnings
+@load ./extracted_file_limits.bro

scripts/policy/tuning/defaults/extracted_file_limits.bro

@@ -0,0 +1,4 @@
+@load base/files/extract
+
+# 100 MB.
+redef FileExtract::default_limit = 104857600;

scripts/test-all-policy.bro

@@ -89,6 +89,7 @@
 @load protocols/ssl/validate-certs.bro
 @load tuning/__load__.bro
 @load tuning/defaults/__load__.bro
+@load tuning/defaults/extracted_file_limits.bro
 @load tuning/defaults/packet-fragments.bro
 @load tuning/defaults/warnings.bro
 @load tuning/logs-to-elasticsearch.bro

src/analyzer/protocol/dnp3/dnp3-protocol.pac

@@ -112,10 +112,10 @@ type Request_Objects(function_code: uint8) = record {
 type Response_Objects(function_code: uint8) = record {
 	object_header: Object_Header(function_code);
 	data: case (object_header.object_type_field) of {
-		0x0101 -> biwoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) ];
+		0x0101 -> biwoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
-		0x0301 -> diwoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) ];
+		0x0301 -> diwoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
-		0x0a01 -> bowoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) ];
+		0x0a01 -> bowoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
-		0x0c03 -> bocmd_PM: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) ];
+		0x0c03 -> bocmd_PM: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
 		default -> ojbects: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item];
 	};
 };

src/bro.bif

@@ -3897,8 +3897,8 @@ function flush_all%(%): bool
 ##
 ## f: The directory name.
 ##
-## Returns: Returns true if the operation succeeds, or false if the
+## Returns: Returns true if the operation succeeds or if *f* already exists,
-##           creation fails or if *f* exists already.
+##          and false if the file creation fails.
 ##
 ## .. bro:see:: active_file open_for_append close write_file
 ##              get_file_name set_buf flush_all enable_raw_output

src/builtin-func.l

@@ -209,8 +209,8 @@ void init_alternative_mode()
 
 	static char guard[1024];
 	getcwd(guard, sizeof(guard));
-	strncat(guard, "/", sizeof(guard));
+	strncat(guard, "/", sizeof(guard) - strlen(guard) - 1);
-	strncat(guard, input_filename, sizeof(guard));
+	strncat(guard, input_filename, sizeof(guard) - strlen(guard) - 1);
 
 	for ( char* p = guard; *p; p++ )
 		{

src/file_analysis/AnalyzerSet.cc

@@ -35,6 +35,14 @@ AnalyzerSet::~AnalyzerSet()
 	delete analyzer_hash;
 	}
 
+Analyzer* AnalyzerSet::Find(file_analysis::Tag tag, RecordVal* args)
+	{
+	HashKey* key = GetKey(tag, args);
+	Analyzer* rval = analyzer_map.Lookup(key);
+	delete key;
+	return rval;
+	}
+
 bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args)
 	{
 	HashKey* key = GetKey(tag, args);

src/file_analysis/AnalyzerSet.h

@@ -37,6 +37,14 @@ public:
 	 */
 	~AnalyzerSet();
 
+	/**
+	 * Looks up an analyzer by its tag and arguments.
+	 * @param tag an analyzer tag.
+	 * @param args an \c AnalyzerArgs record.
+	 * @return pointer to an analyzer instance, or a null pointer if not found.
+	 */
+	Analyzer* Find(file_analysis::Tag tag, RecordVal* args);
+
 	/**
 	 * Attach an analyzer to #file immediately.
 	 * @param tag the analyzer tag of the file analyzer to add.

src/file_analysis/File.cc

@@ -14,6 +14,8 @@
 #include "analyzer/Analyzer.h"
 #include "analyzer/Manager.h"
 
+#include "analyzer/extract/Extract.h"
+
 using namespace file_analysis;
 
 static Val* empty_connection_table()
@@ -203,6 +205,22 @@ void File::SetTimeoutInterval(double interval)
 	val->Assign(timeout_interval_idx, new Val(interval, TYPE_INTERVAL));
 	}
 
+bool File::SetExtractionLimit(RecordVal* args, uint64 bytes)
+	{
+	Analyzer* a = analyzers.Find(file_mgr->GetComponentTag("EXTRACT"), args);
+
+	if ( ! a )
+		return false;
+
+	Extract* e = dynamic_cast<Extract*>(a);
+
+	if ( ! e )
+		return false;
+
+	e->SetLimit(bytes);
+	return true;
+	}
+
 void File::IncrementByteCount(uint64 size, int field_idx)
 	{
 	uint64 old = LookupFieldDefaultCount(field_idx);
@@ -458,7 +476,7 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl)
 			}
 		}
 
-	if ( h == file_new || h == file_timeout )
+	if ( h == file_new || h == file_timeout || h == file_extraction_limit )
 		{
 		// immediate feedback is required for these events.
 		mgr.Drain();

src/file_analysis/File.h

@@ -56,6 +56,14 @@ public:
 	 */
 	void SetTimeoutInterval(double interval);
 
+	/**
+	 * Change the maximum size that an attached extraction analyzer is allowed.
+	 * @param args the file extraction analyzer whose limit needs changed.
+	 * @param bytes new limit.
+	 * @return false if no extraction analyzer is active, else true.
+	 */
+	bool SetExtractionLimit(RecordVal* args, uint64 bytes);
+
 	/**
 	 * @return value of the "id" field from #val record.
 	 */

src/file_analysis/Manager.cc

@@ -184,6 +184,17 @@ bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
 	return true;
 	}
 
+bool Manager::SetExtractionLimit(const string& file_id, RecordVal* args,
+                                 uint64 n) const
+	{
+	File* file = LookupFile(file_id);
+
+	if ( ! file )
+		return false;
+
+	return file->SetExtractionLimit(args, n);
+	}
+
 bool Manager::AddAnalyzer(const string& file_id, file_analysis::Tag tag,
                           RecordVal* args) const
 	{

src/file_analysis/Manager.h

@@ -173,6 +173,19 @@ public:
 	 */
 	bool SetTimeoutInterval(const string& file_id, double interval) const;
 
+	/**
+	 * Sets a limit on the maximum size allowed for extracting the file
+	 * to local disk;
+	 * @param file_id the file identifier/hash.
+	 * @param args a \c AnalyzerArgs value which describes a file analyzer,
+	 *        which should be a file extraction analyzer.
+	 * @param n the new extraction limit, in bytes.
+	 * @return false if file identifier and analyzer did not map to anything,
+	 *         else true.
+	 */
+	bool SetExtractionLimit(const string& file_id, RecordVal* args,
+	                        uint64 n) const;
+
 	/**
 	 * Queue attachment of an analzer to the file identifier.  Multiple
 	 * analyzers of a given type can be attached per file identifier at a time

src/file_analysis/analyzer/data_event/Plugin.cc

@@ -1,26 +1,8 @@
 #include "plugin/Plugin.h"
-#include "file_analysis/Component.h"
 
 #include "DataEvent.h"
 
-namespace plugin { namespace Bro_FileDataEvent {
+BRO_PLUGIN_BEGIN(Bro, FileDataEvent)
-
+	BRO_PLUGIN_DESCRIPTION("Delivers file content via events");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("DATA_EVENT", DataEvent);
-protected:
+BRO_PLUGIN_END
-	void InitPreScript()
-		{
-		SetName("Bro::FileDataEvent");
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Delivers file content via events");
-
-		AddComponent(new ::file_analysis::Component("DATA_EVENT",
-		        ::file_analysis::DataEvent::Instantiate));
-		}
-};
-
-Plugin __plugin;
-
-} }

src/file_analysis/analyzer/extract/CMakeLists.txt

@@ -5,4 +5,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
 
 bro_plugin_begin(Bro FileExtract)
 bro_plugin_cc(Extract.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_bif(functions.bif)
 bro_plugin_end()

src/file_analysis/analyzer/extract/Extract.cc

@@ -4,13 +4,15 @@
 
 #include "Extract.h"
 #include "util.h"
+#include "Event.h"
 #include "file_analysis/Manager.h"
 
 using namespace file_analysis;
 
-Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
+Extract::Extract(RecordVal* args, File* file, const string& arg_filename,
+                 uint64 arg_limit)
     : file_analysis::Analyzer(file_mgr->GetComponentTag("EXTRACT"), args, file),
-	  filename(arg_filename)
+      filename(arg_filename), limit(arg_limit)
 	{
 	fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
 
@@ -29,15 +31,51 @@ Extract::~Extract()
 		safe_close(fd);
 	}
 
-file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
+static Val* get_extract_field_val(RecordVal* args, const char* name)
 	{
 	using BifType::Record::Files::AnalyzerArgs;
-	Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename"));
+	Val* rval = args->Lookup(AnalyzerArgs->FieldOffset(name));
 
-	if ( ! v )
+	if ( ! rval )
+		reporter->Error("File extraction analyzer missing arg field: %s", name);
+
+	return rval;
+	}
+
+file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
+	{
+	Val* fname = get_extract_field_val(args, "extract_filename");
+	Val* limit = get_extract_field_val(args, "extract_limit");
+
+	if ( ! fname || ! limit )
 		return 0;
 
-	return new Extract(args, file, v->AsString()->CheckString());
+	return new Extract(args, file, fname->AsString()->CheckString(),
+	                   limit->AsCount());
+	}
+
+static bool check_limit_exceeded(uint64 lim, uint64 off, uint64 len, uint64* n)
+	{
+	if ( lim == 0 )
+		{
+		*n = len;
+		return false;
+		}
+
+	if ( off >= lim )
+		{
+		*n = 0;
+		return true;
+		}
+
+	*n = lim - off;
+
+	if ( len > *n )
+		return true;
+	else
+		*n = len;
+
+	return false;
 	}
 
 bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
@@ -45,6 +83,26 @@ bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 	if ( ! fd )
 		return false;
 
-	safe_pwrite(fd, data, len, offset);
+	uint64 towrite = 0;
-	return true;
+	bool limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite);
+
+	if ( limit_exceeded && file_extraction_limit )
+		{
+		File* f = GetFile();
+		val_list* vl = new val_list();
+		vl->append(f->GetVal()->Ref());
+		vl->append(Args()->Ref());
+		vl->append(new Val(limit, TYPE_COUNT));
+		vl->append(new Val(offset, TYPE_COUNT));
+		vl->append(new Val(len, TYPE_COUNT));
+		f->FileEvent(file_extraction_limit, vl);
+
+		// Limit may have been modified by BIF, re-check it.
+		limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite);
+		}
+
+	if ( towrite > 0 )
+		safe_pwrite(fd, data, towrite, offset);
+
+	return ( ! limit_exceeded );
 	}

src/file_analysis/analyzer/extract/Extract.h

@@ -9,6 +9,8 @@
 #include "File.h"
 #include "Analyzer.h"
 
+#include "analyzer/extract/events.bif.h"
+
 namespace file_analysis {
 
 /**
@@ -41,6 +43,13 @@ public:
 	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
 
+	/**
+	 * Sets the maximum allowed extracted file size.  A value of zero means
+	 * "no limit".
+	 * @param bytes number of bytes allowed to be extracted
+	 */
+	void SetLimit(uint64 bytes) { limit = bytes; }
+
 protected:
 
 	/**
@@ -49,12 +58,15 @@ protected:
 	 * @param file the file to which the analyzer will be attached.
 	 * @param arg_filename a file system path which specifies the local file
 	 *        to which the contents of the file will be extracted/written.
+	 * @param arg_limit the maximum allowed file size.
 	 */
-	Extract(RecordVal* args, File* file, const string& arg_filename);
+	Extract(RecordVal* args, File* file, const string& arg_filename,
+	        uint64 arg_limit);
 
 private:
 	string filename;
 	int fd;
+	uint64 limit;
 };
 
 } // namespace file_analysis

src/file_analysis/analyzer/extract/Plugin.cc

@@ -1,26 +1,10 @@
 #include "plugin/Plugin.h"
-#include "file_analysis/Component.h"
 
 #include "Extract.h"
 
-namespace plugin { namespace Bro_FileExtract {
+BRO_PLUGIN_BEGIN(Bro, FileExtract)
-
+	BRO_PLUGIN_DESCRIPTION("Extract file content to local file system");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("EXTRACT", Extract);
-protected:
+	BRO_PLUGIN_BIF_FILE(events);
-	void InitPreScript()
+	BRO_PLUGIN_BIF_FILE(functions);
-		{
+BRO_PLUGIN_END
-		SetName("Bro::FileExtract");
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Extract file content to local file system");
-
-		AddComponent(new ::file_analysis::Component("EXTRACT",
-		        ::file_analysis::Extract::Instantiate));
-		}
-};
-
-Plugin __plugin;
-
-} }

src/file_analysis/analyzer/extract/events.bif

@@ -0,0 +1,19 @@
+## This event is generated when a file extraction analyzer is about
+## to exceed the maximum permitted file size allowed by
+## *extract_size_limit* field of :bro:see:`Files::AnalyzerArgs`.
+## The analyzer is automatically removed from file *f*.
+##
+## f: The file.
+##
+## args: Arguments that identify a particular file extraction analyzer.
+##       This is only provided to be able to pass along to
+##       :bro:see:`FileExtract::set_limit`.
+##
+## limit: The limit, in bytes, the extracted file is about to breach.
+##
+## offset: The offset at which a file chunk is about to be written.
+##
+## len:: The length of the file chunk about to be written.
+##
+## .. bro:see:: Files::add_analyzer Files::ANALYZER_EXTRACT
+event file_extraction_limit%(f: fa_file, args: any, limit: count, offset: count, len: count%);

src/file_analysis/analyzer/extract/functions.bif

@@ -0,0 +1,19 @@
+##! Internal functions used by the extraction file analyzer.
+
+module FileExtract;
+
+%%{
+#include "file_analysis/Manager.h"
+%%}
+
+## :bro:see:`FileExtract::set_limit`.
+function FileExtract::__set_limit%(file_id: string, args: any, n: count%): bool
+    %{
+	using BifType::Record::Files::AnalyzerArgs;
+	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
+    bool result = file_mgr->SetExtractionLimit(file_id->CheckString(), rv, n);
+	Unref(rv);
+    return new Val(result, TYPE_BOOL);
+    %}
+
+module GLOBAL;

src/file_analysis/analyzer/hash/Plugin.cc

@@ -1,33 +1,11 @@
 #include "plugin/Plugin.h"
-#include "file_analysis/Component.h"
 
 #include "Hash.h"
 
-namespace plugin { namespace Bro_FileHash {
+BRO_PLUGIN_BEGIN(Bro, FileHash)
-
+	BRO_PLUGIN_DESCRIPTION("Hash file content");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("MD5", MD5);
-protected:
+	BRO_PLUGIN_FILE_ANALYZER("SHA1", SHA1);
-	void InitPreScript()
+	BRO_PLUGIN_FILE_ANALYZER("SHA256", SHA256);
-		{
+	BRO_PLUGIN_BIF_FILE(events);
-		SetName("Bro::FileHash");
+BRO_PLUGIN_END
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Hash file content");
-
-		AddComponent(new ::file_analysis::Component("MD5",
-		        ::file_analysis::MD5::Instantiate));
-		AddComponent(new ::file_analysis::Component("SHA1",
-		        ::file_analysis::SHA1::Instantiate));
-		AddComponent(new ::file_analysis::Component("SHA256",
-		        ::file_analysis::SHA256::Instantiate));
-
-		extern std::list<std::pair<const char*, int> > __bif_events_init();
-		AddBifInitFunction(&__bif_events_init);
-		}
-};
-
-Plugin __plugin;
-
-} }

src/file_analysis/analyzer/unified2/Plugin.cc

@@ -2,34 +2,11 @@
 
 #include "plugin/Plugin.h"
 
-#include "file_analysis/Component.h"
-
 #include "Unified2.h"
 
-namespace plugin { namespace Bro_Unified2 {
+BRO_PLUGIN_BEGIN(Bro, Unified2)
-
+	BRO_PLUGIN_DESCRIPTION("Analyze Unified2 alert files.");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("UNIFIED2", Unified2);
-protected:
+	BRO_PLUGIN_BIF_FILE(events);
-	void InitPreScript()
+	BRO_PLUGIN_BIF_FILE(types);
-		{
+BRO_PLUGIN_END
-		SetName("Bro::Unified2");
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Analyze Unified2 alert files.");
-
-		AddComponent(new ::file_analysis::Component("UNIFIED2",
-		        ::file_analysis::Unified2::Instantiate));
-
-		extern std::list<std::pair<const char*, int> > __bif_events_init();
-		AddBifInitFunction(&__bif_events_init);
-
-		extern std::list<std::pair<const char*, int> > __bif_types_init();
-		AddBifInitFunction(&__bif_types_init);
-		}
-};
-
-Plugin __plugin;
-
-} }

src/input/readers/Raw.cc

@@ -19,7 +19,12 @@ using threading::Value;
 using threading::Field;
 
 const int Raw::block_size = 4096; // how big do we expect our chunks of data to be.
+pthread_mutex_t Raw::fork_mutex;
 
+bool Raw::ClassInit()
+	{
+	return pthread_mutex_init(&fork_mutex, 0) == 0;
+	}
 
 Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend)
 	{
@@ -77,10 +82,51 @@ void Raw::DoClose()
 		}
 	}
 
+void Raw::ClosePipeEnd(int i)
+	{
+	if ( pipes[i] == -1 )
+		return;
+
+	safe_close(pipes[i]);
+	pipes[i] = -1;
+	}
+
+bool Raw::LockForkMutex()
+	{
+	int res = pthread_mutex_lock(&fork_mutex);
+	if ( res == 0 )
+		return true;
+
+	Error(Fmt("cannot lock fork mutex: %d", res));
+	return false;
+	}
+
+bool Raw::UnlockForkMutex()
+	{
+	int res = pthread_mutex_unlock(&fork_mutex);
+	if ( res == 0 )
+		return true;
+
+	Error(Fmt("cannot unlock fork mutex: %d", res));
+	return false;
+	}
+
 bool Raw::Execute()
 	{
+	// AFAICT, pipe/fork/exec should be thread-safe, but actually having
+	// multiple threads set up pipes and fork concurrently sometimes
+	// results in problems w/ a stdin pipe not ever getting an EOF even
+	// though both ends of it are closed.  But if the same threads
+	// allocate pipes and fork individually or sequentially, that issue
+	// never crops up... ("never" meaning I haven't seen in it in
+	// hundreds of tests using 50+ threads where before I'd see the issue
+	// w/ just 2 threads ~33% of the time).
+	if ( ! LockForkMutex() )
+		return false;
+
 	if ( pipe(pipes) != 0 || pipe(pipes+2) || pipe(pipes+4) )
 		{
+		UnlockForkMutex();
 		Error(Fmt("Could not open pipe: %d", errno));
 		return false;
 		}
@@ -88,6 +134,7 @@ bool Raw::Execute()
 	childpid = fork();
 	if ( childpid < 0 )
 		{
+		UnlockForkMutex();
 		Error(Fmt("Could not create child process: %d", errno));
 		return false;
 		}
@@ -95,65 +142,83 @@ bool Raw::Execute()
 	else if ( childpid == 0 )
 		{
 		// we are the child.
-		safe_close(pipes[stdout_in]);
+		close(pipes[stdout_in]);
 		if ( dup2(pipes[stdout_out], stdout_fileno) == -1 )
-			Error(Fmt("Error on dup2 stdout_out: %d", errno));
+			_exit(252);
 
-		if ( stdin_towrite )
+		close(pipes[stdout_out]);
-			{
-			safe_close(pipes[stdin_out]);
-			if ( dup2(pipes[stdin_in], stdin_fileno) == -1 )
-				Error(Fmt("Error on dup2 stdin_in: %d", errno));
-			}
 
-		if ( use_stderr )
+		close(pipes[stdin_out]);
-			{
+		if ( stdin_towrite && dup2(pipes[stdin_in], stdin_fileno) == -1 )
-			safe_close(pipes[stderr_in]);
+			_exit(253);
-			if ( dup2(pipes[stderr_out], stderr_fileno) == -1 )
+
-				Error(Fmt("Error on dup2 stderr_out: %d", errno));
+		close(pipes[stdin_in]);
-			}
+
+		close(pipes[stderr_in]);
+		if ( use_stderr && dup2(pipes[stderr_out], stderr_fileno) == -1 )
+			_exit(254);
+
+		close(pipes[stderr_out]);
 
 		execl("/bin/sh", "sh", "-c", fname.c_str(), (char*) NULL);
 		fprintf(stderr, "Exec failed :(......\n");
-		exit(255);
+		_exit(255);
 		}
 	else
 		{
 		// we are the parent
-		safe_close(pipes[stdout_out]);
+		if ( ! UnlockForkMutex() )
-		pipes[stdout_out] = -1;
+			return false;
+
+		ClosePipeEnd(stdout_out);
 
 		if ( Info().mode == MODE_STREAM )
 			fcntl(pipes[stdout_in], F_SETFL, O_NONBLOCK);
 
+		ClosePipeEnd(stdin_in);
+
 		if ( stdin_towrite )
-			{
+			// Ya, just always set this to nonblocking. we do not
-			safe_close(pipes[stdin_in]);
+			// want to block on a program receiving data. Note
-			pipes[stdin_in] = -1;
+			// that there is a small gotcha with it. More data is
-			fcntl(pipes[stdin_out], F_SETFL, O_NONBLOCK); // ya, just always set this to nonblocking. we do not want to block on a program receiving data.
+			// queued when more data is read from the program
-			// note that there is a small gotcha with it. More data is queued when more data is read from the program output. Hence, when having
+			// output. Hence, when having a program in
-			// a program in mode_manual where the first write cannot write everything, the rest will be stuck in a queue that is never emptied.
+			// mode_manual where the first write cannot write
-			}
+			// everything, the rest will be stuck in a queue that
+			// is never emptied.
+			fcntl(pipes[stdin_out], F_SETFL, O_NONBLOCK);
+		else
+			ClosePipeEnd(stdin_out);
+
+		ClosePipeEnd(stderr_out);
 
 		if ( use_stderr )
-			{
-			safe_close(pipes[stderr_out]);
-			pipes[stderr_out] = -1;
 			fcntl(pipes[stderr_in], F_SETFL, O_NONBLOCK); // true for this too.
-			}
+		else
+			ClosePipeEnd(stderr_in);
 
 		file = fdopen(pipes[stdout_in], "r");
+
+		if ( ! file )
+			{
+			Error("Could not convert stdout_in fileno to file");
+			return false;
+			}
+
 		pipes[stdout_in] = -1; // will be closed by fclose
 
 		if ( use_stderr )
+			{
 			stderrfile = fdopen(pipes[stderr_in], "r");
-			pipes[stderr_in] = -1; // will be closed by fclose
+
-			if ( file == 0 || (stderrfile == 0 && use_stderr) )
+			if ( ! stderrfile )
 				{
-				Error("Could not convert fileno to file");
+				Error("Could not convert stderr_in fileno to file");
 				return false;
 				}
 
+			pipes[stderr_in] = -1; // will be closed by fclose
+			}
 
 		return true;
 		}
@@ -172,6 +237,7 @@ bool Raw::OpenInput()
 			Error(Fmt("Init: cannot open %s", fname.c_str()));
 			return false;
 			}
+
 		fcntl(fileno(file),  F_SETFD, FD_CLOEXEC);
 		}
 
@@ -194,14 +260,10 @@ bool Raw::CloseInput()
 	if ( use_stderr )
 		fclose(stderrfile);
 
-	if ( execute ) // we do not care if any of those fails. They should all be defined.
+	if ( execute )
 		{
 		for ( int i = 0; i < 6; i ++ )
-			if ( pipes[i] != -1 )
+			ClosePipeEnd(i);
-				{
-				safe_close(pipes[i]);
-				pipes[i] = -1;
-				}
 		}
 
 	file = 0;
@@ -314,7 +376,7 @@ int64_t Raw::GetLine(FILE* arg_file)
 
 	int repeats = 1;
 
-	for (;;)
+	for ( ;; )
 		{
 		size_t readbytes = fread(buf+bufpos+offset, 1, block_size-bufpos, arg_file);
 		pos += bufpos + readbytes;
@@ -402,10 +464,7 @@ void Raw::WriteToStdin()
 		}
 
 	if ( stdin_towrite == 0 ) // send EOF when we are done.
-		{
+		ClosePipeEnd(stdin_out);
-		safe_close(pipes[stdin_out]);
-		pipes[stdin_out] = -1;
-		}
 
 	if ( Info().mode == MODE_MANUAL && stdin_towrite != 0 )
 		{

src/input/readers/Raw.h

@@ -4,6 +4,7 @@
 #define INPUT_READERS_RAW_H
 
 #include <vector>
+#include <pthread.h>
 
 #include "../ReaderBackend.h"
 
@@ -20,6 +21,8 @@ public:
 
 	static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); }
 
+	static bool ClassInit();
+
 protected:
 	virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields);
 	virtual void DoClose();
@@ -27,6 +30,10 @@ protected:
 	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
+	void ClosePipeEnd(int i);
+	bool LockForkMutex();
+	bool UnlockForkMutex();
+
 	bool OpenInput();
 	bool CloseInput();
 	int64_t GetLine(FILE* file);
@@ -44,7 +51,6 @@ private:
 	string separator;
 	unsigned int sep_length; // length of the separator
 
-	static const int block_size;
 	int bufpos;
 	char* buf;
 	char* outbuf;
@@ -72,6 +78,8 @@ private:
 		stderr_out = 5
 	};
 
+	static const int block_size;
+	static pthread_mutex_t fork_mutex;
 };
 
 }

src/main.cc

@@ -57,6 +57,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "input/Manager.h"
 #include "logging/Manager.h"
 #include "logging/writers/Ascii.h"
+#include "input/readers/Raw.h"
 #include "analyzer/Manager.h"
 #include "analyzer/Tag.h"
 #include "plugin/Manager.h"
@@ -842,6 +843,8 @@ int main(int argc, char** argv)
 
 	init_event_handlers();
 
+	input::reader::Raw::ClassInit();
+
 	// The leak-checker tends to produce some false
 	// positives (memory which had already been
 	// allocated before we start the checking is
@@ -1151,10 +1154,10 @@ int main(int argc, char** argv)
 		curl_global_cleanup();
 #endif
 
-		sqlite3_shutdown();
-
 		terminate_bro();
 
+		sqlite3_shutdown();
+
 		// Close files after net_delete(), because net_delete()
 		// might write to connection content files.
 		BroFile::CloseCachedFiles();

src/plugin/Macros.h

@@ -9,6 +9,7 @@
 #define PLUGIN_MACROS_H
 
 #include "analyzer/Component.h"
+#include "file_analysis/Component.h"
 
 /**
  * The current plugin API version. Plugins that won't match this version will
@@ -91,6 +92,19 @@
 #define BRO_PLUGIN_ANALYZER(tag, cls) \
 	AddComponent(new ::analyzer::Component(tag, ::analyzer::cls::InstantiateAnalyzer));
 
+/**
+ * Defines a component implementing a file analyzer.
+ *
+ * @param tag A string with the analyzer's tag. This must be unique across
+ * all loaded analyzers and will translate into a corresponding \c ANALYZER_*
+ * constant at the script-layer.
+ *
+ * @param cls The class that implements the analyzer. It must be derived
+ * (directly or indirectly) from file_analysis::Analyzer.
+ */
+#define BRO_PLUGIN_FILE_ANALYZER(tag, cls) \
+	AddComponent(new ::file_analysis::Component(tag, ::file_analysis::cls::Instantiate));
+
 /**
  * Defines a component implementing a protocol analyzer class that will
  * not be instantiated dynamically. This is for two use-cases: (1) abstract

testing/btest/Baseline/core.leaks.ip-in-ip/output

@@ -1,13 +0,0 @@
-new_connection: tunnel
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-new_connection: tunnel
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf], [cid=[orig_h=babe::beef, orig_p=0/unknown, resp_h=dead::babe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=arKYeMETxOg]]
-new_connection: tunnel
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-tunnel_changed:
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    old: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-    new: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=k6kgXLOoSKl]]

testing/btest/Baseline/core.leaks.ipv6_ext_headers/output

@@ -1,4 +0,0 @@
-weird routing0_hdr from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:78:1:32::2
-[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=53/udp, resp_h=2001:78:1:32::2, resp_p=53/udp]
-[ip=<uninitialized>, ip6=[class=0, flow=0, len=59, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>, mobility=<uninitialized>], [id=43, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=[nxt=17, len=4, rtype=0, segleft=2, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>, mobility=<uninitialized>]]], tcp=<uninitialized>, udp=[sport=53/udp, dport=53/udp, ulen=11], icmp=<uninitialized>]
-[2001:78:1:32::1, 2001:78:1:32::2]

testing/btest/Baseline/core.leaks.vector-val-bifs/output

@@ -1,10 +0,0 @@
-[1, 3, 0, 2]
-[2374950123]
-[1, 3, 0, 2]
-[2374950123]
-[1, 3, 0, 2]
-[2374950123]
-[1, 3, 0, 2]
-[3353991673]
-[1, 3, 0, 2]
-[3353991673]

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -25,6 +25,8 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_DNP3.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DNS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_File.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -25,6 +25,8 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_DNP3.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DNS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_File.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro

testing/btest/Baseline/scripts.base.files.extract.limit/1.out

@@ -0,0 +1 @@
+file_extraction_limit, 3000, 2896, 1448

testing/btest/Baseline/scripts.base.files.extract.limit/2.out

@@ -0,0 +1,3 @@
+file_extraction_limit, 3000, 2896, 1448
+T
+file_extraction_limit, 6000, 5792, 1448

testing/btest/Baseline/scripts.base.files.extract.limit/3.out

@@ -0,0 +1,2 @@
+file_extraction_limit, 7000, 5792, 1448
+T

testing/btest/Baseline/scripts.base.files.extract.limit/extract_files.1

@@ -0,0 +1,72 @@
+The National Center for Supercomputing Applications                     1/28/92
+Anonymous FTP Server General Information
+
+This file contains information about the general structure, as well as
+information on how to obtain files and documentation from the FTP server.
+NCSA software and documentation can also be obtained through the the U.S.
+Mail.  Instructions are included for using this method as well.
+
+Information about the Software Development Group and NCSA software can be 
+found in the /ncsapubs directory in a file called TechResCatalog.
+
+
+THE UNIVERSITY OF ILLINOIS GIVES NO WARRANTY, EXPRESSED OR IMPLIED, FOR THE
+SOFTWARE AND/OR DOCUMENTATION PROVIDED, INCLUDING, WITHOUT LIMITATION, 
+WARRANTY OF MERCHANTABILITY AND WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE.
+
+
+_____________________________________________________________
+
+FTP INSTRUCTIONS
+
+Most NCSA Software is released into the public domain.  That is, for these 
+programs, the public domain has all rights for future licensing, resale, 
+and publication of available packages. If you are connected to Internet
+(NSFNET, ARPANET, MILNET, etc) you may download NCSA software and documentation and source code if it is available, at no charge from the anonymous file 
+transfer protocol (FTP) server at NCSA where you got this file. The procedure
+you should follow to do so is presented  below. If you have any questions
+regarding this procedure or whether you are connected to Internet, consult your local system administration or network expert.
+
+1. Log on to a host at your site that is connected to the Internet and is
+   running software supporting the FTP command.
+
+2. Invoke FTP on most systems by entering the Internet address of the server.
+   Type the following at the shell (usually "%") prompt:
+
+      % ftp ftp.ncsa.uiuc.edu
+
+3. Log in by entering anonymous for the name.
+
+4. Enter your local email address (login@host) for the password.
+
+5. Enter the following at the "ftp>" prompt to copy a text file from our 
+   server to your local host:
+
+      ftp> get filename
+
+   where "filename" is the name of the file you want a copy of.  For example,
+   to get a copy of this file from the server enter:
+
+      ftp> get README.FIRST
+
+   To get a copy of our software brochure, enter:
+
+      ftp> cd ncsapubs
+	   get TechResCatalog 
+
+   NOTE:  Some of the filenames on the server are rather long to aid in
+          identification.  Some operating systems may have problems with names
+          this long.  To change the name the file will have on your local
+          machine type the following at the "ftp>" prompt ("remoteName" is the
+          name of the file on the server and "localName" is the name you want
+          the file to have on your local machine):
+
+             ftp> get remoteName localName
+
+          Example:
+
+             ftp> get TechResCatalog catalog.txt
+
+
+6. For files that are not text files (almost everything else) you will need to
+   specify that you want to transfer binary files.  Do this by ty

testing/btest/Baseline/scripts.base.files.extract.limit/extract_files.2

@@ -0,0 +1,157 @@
+The National Center for Supercomputing Applications                     1/28/92
+Anonymous FTP Server General Information
+
+This file contains information about the general structure, as well as
+information on how to obtain files and documentation from the FTP server.
+NCSA software and documentation can also be obtained through the the U.S.
+Mail.  Instructions are included for using this method as well.
+
+Information about the Software Development Group and NCSA software can be 
+found in the /ncsapubs directory in a file called TechResCatalog.
+
+
+THE UNIVERSITY OF ILLINOIS GIVES NO WARRANTY, EXPRESSED OR IMPLIED, FOR THE
+SOFTWARE AND/OR DOCUMENTATION PROVIDED, INCLUDING, WITHOUT LIMITATION, 
+WARRANTY OF MERCHANTABILITY AND WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE.
+
+
+_____________________________________________________________
+
+FTP INSTRUCTIONS
+
+Most NCSA Software is released into the public domain.  That is, for these 
+programs, the public domain has all rights for future licensing, resale, 
+and publication of available packages. If you are connected to Internet
+(NSFNET, ARPANET, MILNET, etc) you may download NCSA software and documentation and source code if it is available, at no charge from the anonymous file 
+transfer protocol (FTP) server at NCSA where you got this file. The procedure
+you should follow to do so is presented  below. If you have any questions
+regarding this procedure or whether you are connected to Internet, consult your local system administration or network expert.
+
+1. Log on to a host at your site that is connected to the Internet and is
+   running software supporting the FTP command.
+
+2. Invoke FTP on most systems by entering the Internet address of the server.
+   Type the following at the shell (usually "%") prompt:
+
+      % ftp ftp.ncsa.uiuc.edu
+
+3. Log in by entering anonymous for the name.
+
+4. Enter your local email address (login@host) for the password.
+
+5. Enter the following at the "ftp>" prompt to copy a text file from our 
+   server to your local host:
+
+      ftp> get filename
+
+   where "filename" is the name of the file you want a copy of.  For example,
+   to get a copy of this file from the server enter:
+
+      ftp> get README.FIRST
+
+   To get a copy of our software brochure, enter:
+
+      ftp> cd ncsapubs
+	   get TechResCatalog 
+
+   NOTE:  Some of the filenames on the server are rather long to aid in
+          identification.  Some operating systems may have problems with names
+          this long.  To change the name the file will have on your local
+          machine type the following at the "ftp>" prompt ("remoteName" is the
+          name of the file on the server and "localName" is the name you want
+          the file to have on your local machine):
+
+             ftp> get remoteName localName
+
+          Example:
+
+             ftp> get TechResCatalog catalog.txt
+
+
+6. For files that are not text files (almost everything else) you will need to
+   specify that you want to transfer binary files.  Do this by typing the
+   following at the "ftp>" prompt:
+
+      ftp> type binary
+
+   You can now use the "get" command to download binary files.  To switch back
+   to ASCII text transfers type:
+
+      ftp> type ascii
+
+7. The "ls" and "cd" commands can be used at the "ftp>" prompt to list and
+   change directories as in the shell.
+
+8. Enter "quit" or "bye" to exit FTP and return to your local host.
+
+
+_____________________________________________________________
+
+FTP SOFTWARE BY MAIL
+
+To obtain an order form, send your request to the following address:
+
+FTP Archive Tapes
+c/o Debbie Shirley
+152 Computing Applications Building
+605 East Springfield Avenue
+Champaign, IL  61820
+
+or call:
+Debbie at (217) 244-4130
+
+
+_____________________________________________________________
+
+VIRUS INFORMATION
+
+The Software Development Group at NCSA is very virus-conscious. We routinely
+check our machines for viruses and recommend that you do so also. For the
+Macintoshes we use Disinfectant. You can obtain a copy of Disinfectant from
+the /Mac/Utilities directory.
+
+If you use Microsoft DOS or Windows you can find the latest virus scan from 
+the anonymous site oak.oakland.edu in the /SimTel/msdos/virus directory.
+
+_____________________________________________________________
+
+GENERAL INFORMATION
+
+
+DIRECTORY STRUCTURE
+
+The FTP server is organized as specified below:
+
+   /Mac       		Macintosh software
+   /PC        		IBM PC software
+   /Unix      		Software for machines running UNIX or equivalent OS
+   /Unix/SGI		Software that primarily runs on Silicon Graphics 
+			 machines only
+   /Visualization	Software tools for data visualization.
+   /Web			World Wide Web tools, including Mosaic, httpd,
+			and html editors.
+   /HDF   	 	Hierarchical Data Format applications and tools
+   /Samples   		Samples that can be used with most of NCSA software 
+			 tools
+   /Documentation 	Currently being constructed, check each application's 
+			 directory for documentation
+   /ncsapubs		Information produced by the Publications group,
+			 including Metacenter announcements, data link & access,
+			 a software listing, start-up guides, and other 
+			 reference documents.
+   /misc      		Miscellaneous documentation and software
+   /incoming  		directory for contributions
+   /outgoing		swap directory
+
+Information for a particular application can be found in the README file,
+located in the same directory as the application.  The README files contain
+information on new features, known bugs, compile information, and other
+important notes.
+
+All directories on the FTP server contain an INDEX file.  These files outline
+the hierarchical structure of the directory and (recursively) all files and
+directories contained within it.  The INDEX at the root level contains the
+structure of the enire server listing all files and directories on it.  The
+INDEX file in each software directory contains additional information about
+each file.  The letter in parenthesis after the file name indicates how the
+file should be downloaded

testing/btest/Baseline/scripts.base.files.extract.limit/extract_files.3

@@ -0,0 +1,425 @@
+The National Center for Supercomputing Applications                     1/28/92
+Anonymous FTP Server General Information
+
+This file contains information about the general structure, as well as
+information on how to obtain files and documentation from the FTP server.
+NCSA software and documentation can also be obtained through the the U.S.
+Mail.  Instructions are included for using this method as well.
+
+Information about the Software Development Group and NCSA software can be 
+found in the /ncsapubs directory in a file called TechResCatalog.
+
+
+THE UNIVERSITY OF ILLINOIS GIVES NO WARRANTY, EXPRESSED OR IMPLIED, FOR THE
+SOFTWARE AND/OR DOCUMENTATION PROVIDED, INCLUDING, WITHOUT LIMITATION, 
+WARRANTY OF MERCHANTABILITY AND WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE.
+
+
+_____________________________________________________________
+
+FTP INSTRUCTIONS
+
+Most NCSA Software is released into the public domain.  That is, for these 
+programs, the public domain has all rights for future licensing, resale, 
+and publication of available packages. If you are connected to Internet
+(NSFNET, ARPANET, MILNET, etc) you may download NCSA software and documentation and source code if it is available, at no charge from the anonymous file 
+transfer protocol (FTP) server at NCSA where you got this file. The procedure
+you should follow to do so is presented  below. If you have any questions
+regarding this procedure or whether you are connected to Internet, consult your local system administration or network expert.
+
+1. Log on to a host at your site that is connected to the Internet and is
+   running software supporting the FTP command.
+
+2. Invoke FTP on most systems by entering the Internet address of the server.
+   Type the following at the shell (usually "%") prompt:
+
+      % ftp ftp.ncsa.uiuc.edu
+
+3. Log in by entering anonymous for the name.
+
+4. Enter your local email address (login@host) for the password.
+
+5. Enter the following at the "ftp>" prompt to copy a text file from our 
+   server to your local host:
+
+      ftp> get filename
+
+   where "filename" is the name of the file you want a copy of.  For example,
+   to get a copy of this file from the server enter:
+
+      ftp> get README.FIRST
+
+   To get a copy of our software brochure, enter:
+
+      ftp> cd ncsapubs
+	   get TechResCatalog 
+
+   NOTE:  Some of the filenames on the server are rather long to aid in
+          identification.  Some operating systems may have problems with names
+          this long.  To change the name the file will have on your local
+          machine type the following at the "ftp>" prompt ("remoteName" is the
+          name of the file on the server and "localName" is the name you want
+          the file to have on your local machine):
+
+             ftp> get remoteName localName
+
+          Example:
+
+             ftp> get TechResCatalog catalog.txt
+
+
+6. For files that are not text files (almost everything else) you will need to
+   specify that you want to transfer binary files.  Do this by typing the
+   following at the "ftp>" prompt:
+
+      ftp> type binary
+
+   You can now use the "get" command to download binary files.  To switch back
+   to ASCII text transfers type:
+
+      ftp> type ascii
+
+7. The "ls" and "cd" commands can be used at the "ftp>" prompt to list and
+   change directories as in the shell.
+
+8. Enter "quit" or "bye" to exit FTP and return to your local host.
+
+
+_____________________________________________________________
+
+FTP SOFTWARE BY MAIL
+
+To obtain an order form, send your request to the following address:
+
+FTP Archive Tapes
+c/o Debbie Shirley
+152 Computing Applications Building
+605 East Springfield Avenue
+Champaign, IL  61820
+
+or call:
+Debbie at (217) 244-4130
+
+
+_____________________________________________________________
+
+VIRUS INFORMATION
+
+The Software Development Group at NCSA is very virus-conscious. We routinely
+check our machines for viruses and recommend that you do so also. For the
+Macintoshes we use Disinfectant. You can obtain a copy of Disinfectant from
+the /Mac/Utilities directory.
+
+If you use Microsoft DOS or Windows you can find the latest virus scan from 
+the anonymous site oak.oakland.edu in the /SimTel/msdos/virus directory.
+
+_____________________________________________________________
+
+GENERAL INFORMATION
+
+
+DIRECTORY STRUCTURE
+
+The FTP server is organized as specified below:
+
+   /Mac       		Macintosh software
+   /PC        		IBM PC software
+   /Unix      		Software for machines running UNIX or equivalent OS
+   /Unix/SGI		Software that primarily runs on Silicon Graphics 
+			 machines only
+   /Visualization	Software tools for data visualization.
+   /Web			World Wide Web tools, including Mosaic, httpd,
+			and html editors.
+   /HDF   	 	Hierarchical Data Format applications and tools
+   /Samples   		Samples that can be used with most of NCSA software 
+			 tools
+   /Documentation 	Currently being constructed, check each application's 
+			 directory for documentation
+   /ncsapubs		Information produced by the Publications group,
+			 including Metacenter announcements, data link & access,
+			 a software listing, start-up guides, and other 
+			 reference documents.
+   /misc      		Miscellaneous documentation and software
+   /incoming  		directory for contributions
+   /outgoing		swap directory
+
+Information for a particular application can be found in the README file,
+located in the same directory as the application.  The README files contain
+information on new features, known bugs, compile information, and other
+important notes.
+
+All directories on the FTP server contain an INDEX file.  These files outline
+the hierarchical structure of the directory and (recursively) all files and
+directories contained within it.  The INDEX at the root level contains the
+structure of the enire server listing all files and directories on it.  The
+INDEX file in each software directory contains additional information about
+each file.  The letter in parenthesis after the file name indicates how the
+file should be downloaded:  ascii (a), binary (b), or mac binary (m).
+
+The "misc" directories found in some software tool directories contain
+supplementary code or other information.  Refer to the README file in that
+directory for a description of what is contained within the "misc" directory.
+
+The "contrib" directories contain contributed software.  This directory usually
+contains NCSA source that has been modified by people outside of NCSA as well
+as binaries compiled on different platforms not available to the Software 
+Development Group.  If you have modified NCSA software or would like to share 
+some code please contact the developer of the source so arrangemnts can be 
+made to upload it to the "incoming"  directory.  If you are downloading 
+software from the "contrib" directory please note that this software is not 
+supported by NCSA and has not been checked for viruses (see statement on 
+viruses above).  NCSA may not be held responsible for anything resulting from 
+use of the contributed software.  *** RUN AT YOUR OWN RISK ***
+
+
+FILE NAMES
+
+All file names consist of the name of the tool, the version number, and one or
+more extensions.  The extensions identify what type of information is contained
+in the file, and what format it is in.  For example, here is a list of files in
+the /Mac/DataScope directory:
+
+   DataScope2.0.1.asc.tar.Z
+   DataScope2.0.1.src.sit.hqx
+   DataScope2.0.1.smp.sit.hqx
+   DataScope2.0.1.mac.sit.hqx
+   DataScope2.0.1.msw.sit.hqx
+
+The first three character extension indicates what type of data can be found in
+that file (ASCII documentation, source, samples, etc.).  The other extensions
+indicate what format the files are in.  The extensions ".tar" and ".sit"
+indicate types of archives, and the ".Z" and ".hqx" indicate compression and
+encoding schemes.  (See below for instructions on extracting files that have
+been archived and/or compressed.)  Following are a list of extensions and their
+meanings:
+
+   .sn3   Sun 3 executables
+   .sn4   Sun 4 executables
+   .386   Sun 386i executables
+   .sgi   Silicon Graphics Iris executables
+   .dgl   Silicon Graphics Iris using DGL executables
+   .rs6   IBM RS6000 executables
+   .cv2   Convex 2 executables
+   .cv3   Convex 3 executables
+   .cr2   Cray 2 executables
+   .crY   CrayYMP executables
+   .d31   DEC 3100 executables
+   .m88   Motorola 88k executables
+   .m68   Motorola 68k executables
+   .exe   IBM PC executables
+   .mac   Macintosh executables
+   .src   source code
+   .smp   sample files
+   .asc   ASCII text documentation
+   .msw   Microsoft Word documentation
+   .ps    postscript documentation
+   .man   formatted man page
+   .shar  Bourne shell archive
+   .sit   archive created by Macintosh application, StuffIt
+   .hqx   encoded with Macintosh application, BinHex
+   .sea   Self extracting Macintosh archive
+   .tar   archive created with UNIX tar command
+   .Z     compressed with UNIX compress command
+
+The files in the PC directory are the only exception to this naming convention.
+In order to conform with the DOS convention of eight character file names and
+one, three character extension, the names for PC files are slightly different.
+Whenever possible the scheme outlined above is used, but the names are usually
+abbreviated and all but one of the dots "." have been omitted.
+
+
+_______________________________________________________________________________
+EXTRACTING ARCHIVED FILES
+
+
+INSTRUCTIONS FOR MACINTOSH FILES
+
+If a file ends with the extension ".sit" it must be unstuffed with either the
+shareware program StuffIt or the Public Domain program UnStuffIt.  Files ending
+with the ".hqx" must be decoded with BinHex.  These programs can be found on
+the FTP server in the /Mac/Utilities directory.  Note that the BinHex program
+must be downloaded with MacBinary enabled, and the StuffIt program must be
+decoded before it can be used.  Files downloaded from the server may be both
+Stuffed (".sit" extension) and BinHexed (".hqx" extension).  These files must
+be first decoded and then unstuffed.
+
+To decode a file with the ".hqx" extension (a BinHexed file):
+
+   1. Download the file to your Macintosh.
+   2. Start the application BinHex by double-clicking on it.
+   3. From the "File" menu in BinHex, choose "UpLoad -> Application".
+   4. Choose the ".hqx" file to be decoded and select "Open".
+   5. The suggested file name will appear in a dialog box.
+   6. Select "Save" to decode the file.
+
+To uncompress a file with the ".sit" extension (a Stuffed file):
+
+   1. Download the file to your Macintosh.
+   2. Start the application Stuffit by double-clicking on it.
+   3. From the "File" menu in Stuffit, choose "Open Archive...".
+   4. Choose the ".sit" file to be unstuffed and select "Open".  A window with
+      all the files contained in the stuffed file will appear.
+   5. Choose "Select All" in the "Edit" menu to select all of the files.
+   6. Click on the "Extract" box at the bottom of the window.
+   7. Select "Save All" in the dialog box to save all the selected files in
+      the current directory.
+
+
+INSTRUCTIONS FOR PC FILES
+
+Most IBM PC files are archived and compressed using the pkzip utility.
+(If you do not have the pkzip utility on your PC, you may obtain it from the
+FTP server by anonymous ftp.  The file you need is called pkz110.exe and it
+is located in /PC/Telnet/contributions.  Set the ftp mode to binary and "get"
+the file pkz110.exe.  Then, on your PC, run PKZ110.EXE with no arguments and
+several files will be self-extracted, including one called PKUNZIP.EXE.  It
+may then be convenient to copy PKUNZIP.EXE to the directory where you have
+placed, or are going to place, your Telnet files.)
+To extract these files, first download the file with the ".zip" extension to
+your PC and then type the following at the DOS prompt:
+
+   > pkunzip -d filename.zip
+
+where "filename" is the name of the file you want to unarchive.
+
+
+INSTRUCTIONS FOR UNIX FILES
+
+Most files on the FTP server will be both tarred and compressed.  For more
+information on the "tar" and "compress" commands you can type "man tar" and
+"man compress" at your shell prompt to see the online manual page for these
+commands, or ask your system administrator for help.  You should first
+uncompress and then unarchive files ending in ".tar.Z" with the following
+procedure.
+
+Files with the ".Z" extension have been compressed with the UNIX "compress"
+command.  To uncompress these files type the following at the shell prompt:
+
+   % uncompress filename.Z
+
+where "filename.Z" is the name of the file ending with the ".Z" extension that
+you wish to uncompress.
+
+Files with the ".tar" extension have been archived with the UNIX "tar" command.
+To extract the files type the following at the shell prompt:
+
+   % tar xf filename.tar
+
+Some files are archived using a shell archive utility and are indicated as such
+with the ".shar" extension.  To extract the files type the following at the
+shell prompt:
+
+   % sh filename.shar
+
+
+_______________________________________________________________________________
+DOCUMENTATION
+
+NCSA offers users several documentation formats for its programs including
+ASCII text, Microsoft Word, and postscript.  If one of these formats does not
+fit your needs, documentaion can be obtained through the mail at the following
+address:
+
+Documentation Orders
+c/o Debbie Shirley
+152 Computing Applications Building
+605 East Springfield Avenue
+Champaign, IL  61820
+
+or call:
+
+(217) 244-4130
+
+Members of the Software Development Group within NCSA are currently working 
+on videotapes that demonstrate and also offer tutorials for NCSA programs. A
+note will be posted here when these tapes are available for distribution.
+
+
+ASCII FORMAT
+
+ASCII text files are provided for all software and are indicated with the
+".asc" extension.  Helpful figures and diagrams obviously cannot be included
+in this form of documentation.  We suggest you use the other forms of
+documentation if possible.
+
+
+MICROSOFT WORD FORMAT
+
+If you are a Macintosh user, please download documents with the ".msw"
+extension. These files should also be stuffed and BinHexed (information on
+extracting these files from the archive is contained earlier in this file).
+The documents can be previewed and printed using the Microsoft Word
+application.  Word documents contain text, images, and formatting.
+
+
+POSTSCRIPT FORMAT
+
+If you are a UNIX user and/or have access to a postscript printer, please
+download files with the ".pos" extension.  The documents can be previewed using
+a poscript previewer or can be printed directly to a poscript printer using a
+command like "lpr".
+
+
+_______________________________________________________________________________
+BUG REPORTS AND SUPPORT
+
+The Software Development Group at NCSA is very interested in how the software 
+tools developed here are being used. Please send any comments or suggestions 
+you may have to the appropriate address.
+
+NOTE: This is a new kind of shareware. You share your science and
+successes with us, and we can get more resources to share more
+NCSA software with you.
+
+If you want to see more NCSA software, please send us a letter,
+ email or US Mail, telling us what you are doing with our software.
+We need to know:
+
+	(1) What science you are working on - an abstract of your 
+	    work would be fine.
+
+	(2) How NCSA software has helped you, for example, by increasing
+	    your productivity or allowing you to do things you could
+	    not do before.
+
+We encourage you to cite the use of any NCSA software you have used in
+your publications. A bibliography of your work would be extremely 
+helpful.
+
+
+NCSA Telnet for the Macintosh:  Please allow ***time*** for a response.
+
+Bug reports, questions, suggestions may be sent to the addresses below.
+
+        mactelnet@ncsa.uiuc.edu (Internet)
+
+NCSA Telnet for PCs:   Please allow ***time*** for a response.
+
+Bug reports, questions, suggestions may be sent to: 
+        pctelnet@ncsa.uiuc.edu (Internet)
+
+All other NCSA software: 
+
+Bug reports should be emailed to the adresses below.  Be sure to check the
+BUGS NOTES section of the README file before sending email.   
+Please allow ***time*** for a response.
+
+        bugs@ncsa.uiuc.edu (Internet)
+
+
+Questions regarding NCSA developed software tools may be sent to the address
+below.  Please allow ***time*** for a response.
+
+        softdev@ncsa.uiuc.edu (Internet)
+_______________________________________________________________________________
+COPYRIGHTS AND TRADEMARKS
+
+Apple
+Motorola
+Digital Equipment Corp.
+Silicon Graphics Inc.
+International Business Machines
+Sun Microsystems
+UNIX
+StuffIt
+Microsoft

testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr

@@ -1,5 +1,4 @@
 error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat
 error: does-not-exist.dat/Input::READER_ASCII: Init failed
-warning: Stream input is already queued for removal. Ignoring remove.
 error: does-not-exist.dat/Input::READER_ASCII: terminating thread
 received termination signal

testing/btest/Baseline/scripts.base.frameworks.input.raw.basic/out

@@ -6,6 +6,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -23,6 +24,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -40,6 +42,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -57,6 +60,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -74,6 +78,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -91,6 +96,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -108,6 +114,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -125,6 +132,7 @@ print outfile, A::s;
 try = try + 1;
 if (8 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }

testing/btest/Baseline/scripts.base.frameworks.input.raw.execute/out

@@ -3,6 +3,7 @@
 print outfile, description;
 print outfile, tpe;
 print outfile, s;
+Input::remove(input);
 close(outfile);
 terminate();
 }, config={

testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/out

@@ -1,36 +1,20 @@
-[source=cat |, reader=Input::READER_RAW, mode=Input::STREAM, name=input2, fields=<no value description>, want_record=F, ev=line
+Input::EVENT_NEW, cat |, input0
-{ 
-print outfile, A::description;
-print outfile, A::tpe;
-print outfile, A::s;
-try = try + 1;
-if (2 == try) 
-{ 
-Input::remove(input2);
-close(outfile);
-terminate();
-}
-
-}, config={
-[stdin] = hello^Jthere^A^B^C^D^E^A^B^Cyay
-}]
-Input::EVENT_NEW
 hello
-[source=cat |, reader=Input::READER_RAW, mode=Input::STREAM, name=input2, fields=<no value description>, want_record=F, ev=line
+Input::EVENT_NEW, cat |, input0
-{ 
+there^A^B^C^D^E^A^B^Cyay0
-print outfile, A::description;
+Input::EVENT_NEW, cat |, input1
-print outfile, A::tpe;
+hello
-print outfile, A::s;
+Input::EVENT_NEW, cat |, input1
-try = try + 1;
+there^A^B^C^D^E^A^B^Cyay01
-if (2 == try) 
+Input::EVENT_NEW, cat |, input2
-{ 
+hello
-Input::remove(input2);
+Input::EVENT_NEW, cat |, input2
-close(outfile);
+there^A^B^C^D^E^A^B^Cyay012
-terminate();
+Input::EVENT_NEW, cat |, input3
-}
+hello
-
+Input::EVENT_NEW, cat |, input3
-}, config={
+there^A^B^C^D^E^A^B^Cyay0123
-[stdin] = hello^Jthere^A^B^C^D^E^A^B^Cyay
+Input::EVENT_NEW, cat |, input4
-}]
+hello
-Input::EVENT_NEW
+Input::EVENT_NEW, cat |, input4
-there^A^B^C^D^E^A^B^Cyay
+there^A^B^C^D^E^A^B^Cyay01234

testing/btest/Baseline/scripts.base.frameworks.input.raw.rereadraw/out

@@ -6,6 +6,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -23,6 +24,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -40,6 +42,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -57,6 +60,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -74,6 +78,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -91,6 +96,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -108,6 +114,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -125,6 +132,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -142,6 +150,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -159,6 +168,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -176,6 +186,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -193,6 +204,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -210,6 +222,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -227,6 +240,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -244,6 +258,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }
@@ -261,6 +276,7 @@ print outfile, A::s;
 try = try + 1;
 if (16 == try) 
 { 
+Input::remove(input);
 close(outfile);
 terminate();
 }

testing/btest/Baseline/scripts.base.protocols.dnp3.dnp3_link_only/coverage

@@ -1 +1 @@
-6 of 51 events triggered by trace
+7 of 51 events triggered by trace

testing/btest/Baseline/scripts.base.protocols.dnp3.events/coverage

@@ -1 +1 @@
-9 of 51 events triggered by trace
+11 of 51 events triggered by trace

testing/btest/Baseline/scripts.base.protocols.dnp3.events/dnp3.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dnp3
-#open	2013-08-12-18-24-03
+#open	2013-08-23-23-05-27
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fc_request	fc_reply	iin
 #types	time	string	addr	port	addr	port	string	string	count
 1097501938.504844	UWkUyAuUGXf	10.0.0.8	2789	10.0.0.3	20000	-	UNSOLICITED_RESPONSE	4096
@@ -72,4 +72,4 @@
 1178206045.032815	3PKsZ2Uye21	192.168.66.33	1167	192.168.66.34	20000	READ	RESPONSE	6
 1178206045.557097	3PKsZ2Uye21	192.168.66.33	1167	192.168.66.34	20000	READ	RESPONSE	6
 1178206046.086403	3PKsZ2Uye21	192.168.66.33	1167	192.168.66.34	20000	READ	RESPONSE	6
-#close	2013-08-12-18-24-03
+#close	2013-08-23-23-05-27

testing/btest/Baseline/scripts.base.protocols.dnp3.events/output

@@ -335,11 +335,50 @@ dnp3_object_header, T, 15361, 6, 0, 65535, 65535
 dnp3_header_block, F, 25605, 78, 68, 3, 4
 dnp3_application_response_header, F, 129, 0
 dnp3_object_header, F, 257, 0, 6, 0, 5
-dnp3_object_header, F, 522, 2, 4294705410, 17104896, 16843009
-dnp3_object_header, F, 276, 5, 0, 0, 21
-dnp3_object_header, F, 2304, 0, 1, 0, 0
 dnp3_object_prefix, F, 0
-dnp3_debug_byte, F, \0\0\0\x1e^C\0\0^F\xc5\0\0\0\xc7\0\0\0\xc8\0\0\0^A\0\0\0%\x1c\0\0^N\x1c\0\0^P\x1c\0\0
+dnp3_response_data_object, F, 2
+dnp3_object_header, F, 2562, 0, 6, 0, 5
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_header, F, 5125, 0, 1, 0, 0
+dnp3_object_prefix, F, 0
+dnp3_counter_32woFlag, F, 0
+dnp3_response_data_object, F, 255
+dnp3_object_header, F, 5385, 0, 1, 0, 0
+dnp3_object_prefix, F, 0
+dnp3_frozen_counter_32woFlag, F, 0
+dnp3_response_data_object, F, 255
+dnp3_object_header, F, 7683, 0, 7, 0, 6
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 197
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 199
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 200
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 1
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 7205
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 7182
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 7184
 dnp3_response_data_object, F, 255
 dnp3_header_block, F, 25605, 10, 68, 6, 4
 dnp3_application_response_header, F, 130, 0
@@ -427,9 +466,50 @@ dnp3_object_header, T, 15361, 6, 0, 65535, 65535
 dnp3_header_block, F, 25605, 78, 68, 3, 4
 dnp3_application_response_header, F, 129, 0
 dnp3_object_header, F, 257, 0, 6, 0, 5
-dnp3_object_header, F, 6410, 2, 2155643138, 2164588544, 25264385
 dnp3_object_prefix, F, 0
-dnp3_debug_byte, F, ^A^T^E\0\0\0 \0\0\0^U^I\0\0\0\0\0\0\0\x1e^C\0\0^F\xca\0\0\0\xcb\0\0\0\xc9\0\0\0\xff\xff\xff\xfff!\0\0Y!\0\0K!\0\0
+dnp3_response_data_object, F, 25
+dnp3_object_header, F, 2562, 0, 6, 0, 5
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 129
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 129
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 129
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_prefix, F, 0
+dnp3_response_data_object, F, 1
+dnp3_object_header, F, 5125, 0, 1, 0, 0
+dnp3_object_prefix, F, 0
+dnp3_counter_32woFlag, F, 32
+dnp3_response_data_object, F, 255
+dnp3_object_header, F, 5385, 0, 1, 0, 0
+dnp3_object_prefix, F, 0
+dnp3_frozen_counter_32woFlag, F, 0
+dnp3_response_data_object, F, 255
+dnp3_object_header, F, 7683, 0, 7, 0, 6
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 202
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 203
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 201
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 18446744073709551615
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 8550
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 8537
+dnp3_response_data_object, F, 255
+dnp3_object_prefix, F, 0
+dnp3_analog_input_32woFlag, F, 8523
 dnp3_response_data_object, F, 255
 dnp3_header_block, T, 25605, 8, 196, 4, 3
 dnp3_application_request_header, T, 14

testing/btest/core/leaks/ayiya.test

@@ -4,4 +4,5 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/tunnels/ayiya3.trace
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/ayiya3.trace
+# @TEST-EXEC: btest-bg-wait 15

testing/btest/core/leaks/dataseries-rotate.bro

@@ -5,7 +5,8 @@
 # @TEST-GROUP: leaks
 # @TEST-GROUP: dataseries
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/rotation.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/rotation.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: btest-bg-wait 15
 
 module Test;
 

testing/btest/core/leaks/dataseries.bro

@@ -7,4 +7,5 @@
 # @TEST-GROUP: dataseries
 #
 # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: btest-bg-wait 15

testing/btest/core/leaks/dns.bro

@@ -4,7 +4,8 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-bg-wait -k 15
 
 const foo: set[addr] = {
      google.com

testing/btest/core/leaks/file-analysis-http-get.bro

@@ -4,7 +4,8 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT
+# @TEST-EXEC: btest-bg-wait 15
 
 redef test_file_analysis_source = "HTTP";
 

testing/btest/core/leaks/gridftp.test

@@ -4,7 +4,8 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/globus-url-copy.trace %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/globus-url-copy.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 15
 
 @load base/protocols/ftp/gridftp
 

testing/btest/core/leaks/gtp_opt_header.test

@@ -4,7 +4,8 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/tunnels/gtp/gtp6_gtp_0x32.pcap %INPUT >out
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/gtp/gtp6_gtp_0x32.pcap %INPUT >out
+# @TEST-EXEC: btest-bg-wait 15
 
 # Some GTPv1 headers have some optional fields totaling to a 4-byte extension
 # of the mandatory header.

testing/btest/core/leaks/hook.bro

@@ -4,7 +4,8 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 15
 
 type rec: record {
 	a: count;

testing/btest/core/leaks/incr-vec-expr.test

@@ -4,7 +4,8 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -b -m -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT
+# @TEST-EXEC: btest-bg-wait 15
 
 type rec: record {
     a: count;

testing/btest/core/leaks/ip-in-ip.test

@@ -4,10 +4,10 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/tunnels/6in6.pcap %INPUT >>output
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro1 bro -m -b -r $TRACES/tunnels/6in6.pcap %INPUT
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/tunnels/6in6in6.pcap %INPUT >>output
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro2 bro -m -b -r $TRACES/tunnels/6in6in6.pcap %INPUT
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/tunnels/6in6-tunnel-change.pcap %INPUT >>output
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro3 bro -m -b -r $TRACES/tunnels/6in6-tunnel-change.pcap %INPUT
-# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-bg-wait 15
 
 event new_connection(c: connection)
 	{

testing/btest/core/leaks/ipv6_ext_headers.test

@@ -4,8 +4,8 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/ipv6-hbh-routing0.trace %INPUT >output
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/ipv6-hbh-routing0.trace %INPUT
-# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-bg-wait 15
 
 # Just check that the event is raised correctly for a packet containing
 # extension headers.

testing/btest/core/leaks/string-indexing.bro

@@ -4,7 +4,8 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 15
 
 
 event new_connection(c: connection)

testing/btest/core/leaks/switch-statement.bro

@@ -4,7 +4,8 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 15
 
 type MyEnum: enum {
 	RED,

testing/btest/core/leaks/teredo.bro

@@ -4,7 +4,8 @@
 #
 # @TEST-GROUP: leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/tunnels/Teredo.pcap %INPUT >output
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/tunnels/Teredo.pcap %INPUT >output
+# @TEST-EXEC: btest-bg-wait 15
 
 function print_teredo(name: string, outer: connection, inner: teredo_hdr)
 	{

testing/btest/core/leaks/test-all.bro

@@ -4,4 +4,5 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace test-all-policy
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -r $TRACES/wikipedia.trace test-all-policy
+# @TEST-EXEC: btest-bg-wait 15

testing/btest/core/leaks/vector-val-bifs.test

@@ -8,8 +8,8 @@
 # assuming that it didn't automatically Ref the VectorType argument and thus
 # leaked that memeory.
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/ftp/ipv4.trace %INPUT >output
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/ftp/ipv4.trace %INPUT
-# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-bg-wait 15
 
 function myfunc(aa: interval, bb: interval): int
     {

testing/btest/scripts/base/files/extract/limit.bro

@@ -0,0 +1,44 @@
+# @TEST-EXEC: bro -b -r $TRACES/ftp/retr.trace %INPUT max_extract=3000 efname=1
+# @TEST-EXEC: btest-diff extract_files/1
+# @TEST-EXEC: btest-diff 1.out
+# @TEST-EXEC: bro -b -r $TRACES/ftp/retr.trace %INPUT max_extract=3000 efname=2 double_it=T
+# @TEST-EXEC: btest-diff extract_files/2
+# @TEST-EXEC: btest-diff 2.out
+# @TEST-EXEC: bro -b -r $TRACES/ftp/retr.trace %INPUT max_extract=7000 efname=3 unlimit_it=T
+# @TEST-EXEC: btest-diff extract_files/3
+# @TEST-EXEC: btest-diff 3.out
+
+@load base/files/extract
+@load base/protocols/ftp
+
+global outfile: file;
+const max_extract: count = 0 &redef;
+const double_it: bool = F &redef;
+const unlimit_it: bool = F &redef;
+const efname: string = "0" &redef;
+global doubled: bool = F;
+
+event file_new(f: fa_file)
+    {
+    Files::add_analyzer(f, Files::ANALYZER_EXTRACT,
+	                    [$extract_filename=efname, $extract_limit=max_extract]);
+    }
+
+event file_extraction_limit(f: fa_file, args: any, limit: count, offset: count, len: count)
+    {
+    print outfile, "file_extraction_limit", limit, offset, len;
+
+	if ( double_it && ! doubled )
+		{
+		doubled = T;
+		print outfile, FileExtract::set_limit(f, args, max_extract*2);
+		}
+
+	if ( unlimit_it )
+		print outfile, FileExtract::set_limit(f, args, 0);
+    }
+
+event bro_init()
+	{
+	outfile = open(fmt("%s.out", efname));
+	}

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 redef exit_only_after_terminate = T;
@@ -50,13 +50,13 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail.
+	Input::remove("ssh");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/bignumber.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 redef exit_only_after_terminate = T; 
@@ -31,12 +31,12 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/binary.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 redef exit_only_after_terminate = T;
@@ -39,6 +39,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, a: string, b
 	try = try + 1;
 	if ( try == 3 )
 		{
+		Input::remove("input");
 		close(outfile);
 		terminate();
 		}
@@ -49,5 +50,4 @@ event bro_init()
 	try = 0;
 	outfile = open("../out");
 	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
-	Input::remove("input");
 	}

testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

@@ -2,7 +2,7 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: cp input2.log input.log
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input1.log

testing/btest/scripts/base/frameworks/input/emptyvals.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -34,12 +34,12 @@ event bro_init()
     outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/event.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -39,12 +39,12 @@ event bro_init()
 	{
 	outfile = open("../out");
 	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, "End-of-data";
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/invalidnumbers.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
@@ -35,11 +35,11 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/invalidtext.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
@@ -33,11 +33,11 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/missing-file.bro

@@ -23,5 +23,4 @@ event bro_init()
 	try = 0;
 	outfile = open("../out");
 	Input::add_event([$source="does-not-exist.dat", $name="input", $fields=Val, $ev=line, $want_record=F]);
-	Input::remove("input");
 	}

testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -32,12 +32,12 @@ event bro_init()
 	{
 	outfile = open("../out");
 	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/onecolumn-record.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -32,12 +32,12 @@ event bro_init()
 	{
 	outfile = open("../out");
 	Input::add_table([$name="input", $source="../input.log", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/optional.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -42,12 +42,12 @@ event bro_init()
 	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
 				$pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; }
 				]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/port.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -37,7 +37,6 @@ event bro_init()
 		print outfile, servers[1.2.3.5];
 	if ( 1.2.3.6 in servers )
 		print outfile, servers[1.2.3.6];
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
@@ -45,6 +44,7 @@ event Input::end_of_data(name: string, source: string)
 	print outfile, servers[1.2.3.4];
 	print outfile, servers[1.2.3.5];
 	print outfile, servers[1.2.3.6];
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/predicate-stream.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 #
 # only difference from predicate.bro is, that this one uses a stream source.
@@ -59,6 +59,7 @@ event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, r
 		print outfile, "VALID";
 	if ( 7 in servers )
 		print outfile, "VALID";
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}
@@ -71,6 +72,5 @@ event bro_init()
 	Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line,
 				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
 				]);
-	Input::remove("input");
 	}
 

testing/btest/scripts/base/frameworks/input/predicate.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -41,7 +41,6 @@ event bro_init()
 	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F,
 				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
 				]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
@@ -60,6 +59,7 @@ event Input::end_of_data(name: string, source: string)
 		print outfile, "VALID";
 	if ( 7 in servers )
 		print outfile, "VALID";
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/predicatemodify.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -45,12 +45,12 @@ event bro_init()
 				return T; 
 				}
 				]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro

@@ -8,7 +8,7 @@
 # @TEST-EXEC: cp input4.log input.log
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: cp input5.log input.log
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 #
 

testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 # Ok, this one tests a fun case.
@@ -42,12 +42,12 @@ event bro_init()
 	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers,
 				$pred(typ: Input::Event, left: Idx, right: Val) = { if ( right$confidence > 90 ) { return T; } return F; }
 				]);
-	Input::remove("input");
 	}
 
 event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/raw/basic.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -32,6 +32,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	try = try + 1;
 	if ( try == 8 )
 		{
+		Input::remove("input");
 		close(outfile);
 		terminate();
 		}
@@ -42,5 +43,4 @@ event bro_init()
 	try = 0;
 	outfile = open("../out");
 	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
-	Input::remove("input");
 	}

testing/btest/scripts/base/frameworks/input/raw/execute.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out
 # @TEST-EXEC: btest-diff out
 
@@ -27,6 +27,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	print outfile, description;
 	print outfile, tpe;
 	print outfile, s;
+	Input::remove("input");
 	close(outfile);
 	terminate();
 	}
@@ -35,5 +36,4 @@ event bro_init()
 	{
 	outfile = open("../out.tmp");
 	Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F]);
-	Input::remove("input");
 	}

testing/btest/scripts/base/frameworks/input/raw/executestdin.bro

@@ -1,13 +1,19 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff test.txt
-# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 redef exit_only_after_terminate = T;
 @load base/frameworks/communication  # let network-time run. otherwise there are no heartbeats...
 
 global outfile: file;
-global try: count;
+global processes_finished: count = 0;
+global n: count = 0;
+global total_processes: count = 0;
+
+global config_strings: table[string] of string = {
+	["stdin"] = "hello\nthere\1\2\3\4\5\1\2\3yay"
+};
 
 module A;
 
@@ -17,27 +23,46 @@ type Val: record {
 
 event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	{
-	print outfile, description;
+	print outfile, tpe, description$source, description$name;
-	print outfile, tpe;
 	print outfile, s;
-	try = try + 1;
+	}
-	if ( try == 2 )
+
+event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool)
+	{
+	print "process_finished", name, source;
+	Input::remove(name);
+	++processes_finished;
+	if ( processes_finished == total_processes )
 		{
-		Input::remove("input2");
 		close(outfile);
 		terminate();
 		}
 	}
 
+function more_input(name_prefix: string)
+	{
+	local name = fmt("%s%d", name_prefix, n);
+	config_strings["stdin"] += fmt("%d", n);
+	++n;
+	++total_processes;
+	Input::add_event([$source="cat |",
+	                  $reader=Input::READER_RAW, $mode=Input::STREAM,
+	                  $name=name, $fields=Val, $ev=line, $want_record=F,
+	                  $config=config_strings]);
+	}
+
 event bro_init()
 	{
-	local config_strings: table[string] of string = {
-		["stdin"] = "hello\nthere\1\2\3\4\5\1\2\3yay"
-		#["stdin"] = "yay"
-	};
-
-	try = 0;
 	outfile = open("../out");
-	Input::add_event([$source="cat > ../test.txt |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
+	++total_processes;
-	Input::add_event([$source="cat |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input2", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
+
+	Input::add_event([$source="cat > ../test.txt |",
+	                  $reader=Input::READER_RAW, $mode=Input::STREAM,
+	                  $name="input", $fields=Val, $ev=line, $want_record=F,
+	                  $config=config_strings]);
+	more_input("input");
+	more_input("input");
+	more_input("input");
+	more_input("input");
+	more_input("input");
 	}

testing/btest/scripts/base/frameworks/input/raw/executestream.bro

@@ -4,7 +4,7 @@
 # @TEST-EXEC: cat input2.log >> input.log
 # @TEST-EXEC: sleep 3
 # @TEST-EXEC: cat input3.log >> input.log
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 redef exit_only_after_terminate = T;

testing/btest/scripts/base/frameworks/input/raw/long.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: dd if=/dev/zero of=input.log bs=8193 count=1
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 #
 # this test should be longer than one block-size. to test behavior of input-reader if it has to re-allocate stuff.
@@ -23,6 +23,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	try = try + 1;
 	if ( try == 1 )
 		{
+		Input::remove("input");
 		close(outfile);
 		terminate();
 		}
@@ -33,5 +34,4 @@ event bro_init()
 	try = 0;
 	outfile = open("../out");
 	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
-	Input::remove("input");
 	}

testing/btest/scripts/base/frameworks/input/raw/rereadraw.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input.log
@@ -32,6 +32,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	try = try + 1;
 	if ( try == 16 )
 		{
+		Input::remove("input");
 		close(outfile);
 		terminate();
 		}
@@ -43,5 +44,4 @@ event bro_init()
 	outfile = open("../out");
 	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::force_update("input");
-	Input::remove("input");
 	}

testing/btest/scripts/base/frameworks/input/raw/stderr.bro

@@ -1,8 +1,9 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 redef exit_only_after_terminate = T;
+@load base/frameworks/communication  # let network-time run. otherwise there are no heartbeats...
 
 type Val: record {
 	s: string;
@@ -37,12 +38,15 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string, i
 		}
 	}
 
+global n = 0;
+
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, "End of Data event";
 	print outfile, name;
-	terminate(); # due to the current design, end_of_data will be called after process_finshed and all line events.
+	++n;
-	# this could potentially change
+	if ( n == 2 )
+		terminate();
 	}
 
 event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool)
@@ -51,6 +55,9 @@ event InputRaw::process_finished(name: string, source:string, exit_code:count, s
 	print outfile, name;
 	if ( exit_code != 0 ) 
 		print outfile, "Exit code != 0";
+	++n;
+	if ( n == 2 )
+		terminate();
 	}
 
 event bro_init()
@@ -62,5 +69,5 @@ event bro_init()
 
 	outfile = open("../out");
 	try = 0;
-	Input::add_event([$source="ls .. ../nonexistant ../nonexistant2 ../nonexistant3 |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
+	Input::add_event([$source="ls .. ../nonexistant ../nonexistant2 ../nonexistant3 |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings, $mode=Input::STREAM]);
 	}

testing/btest/scripts/base/frameworks/input/raw/streamraw.bro

@@ -4,7 +4,7 @@
 # @TEST-EXEC: cat input2.log >> input.log
 # @TEST-EXEC: sleep 3
 # @TEST-EXEC: cat input3.log >> input.log
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 redef exit_only_after_terminate = T;

testing/btest/scripts/base/frameworks/input/repeat.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 @TEST-START-FILE input.log
@@ -36,10 +36,7 @@ event bro_init()
 	try = 0;
 	outfile = open("../out");
 	for ( i in one_to_32 )
-		{
 		Input::add_table([$source="../input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]);
-		Input::remove(fmt("input%d", i));
-		}
 	}
 
 event Input::end_of_data(name: string, source: string)
@@ -47,6 +44,7 @@ event Input::end_of_data(name: string, source: string)
 	print outfile, name;
 	print outfile, source;
 	print outfile, destination;
+	Input::remove(name);
 	try = try + 1;
 	if ( try == 32 )
 		{

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -8,7 +8,7 @@
 # @TEST-EXEC: cp input4.log input.log
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: cp input5.log input.log
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE input1.log

testing/btest/scripts/base/frameworks/input/set.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 @TEST-START-FILE input.log
@@ -32,12 +32,12 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/setseparator.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 @TEST-START-FILE input.log
@@ -32,12 +32,12 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/setspecialcases.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 @TEST-START-FILE input.log
@@ -36,12 +36,12 @@ event bro_init()
 	outfile = open("../out");
 	# first read in the old stuff into the table...
 	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
-	Input::remove("ssh");
 	}
 
 event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
+	Input::remove("ssh");
 	close(outfile);
 	terminate();
 	}

testing/btest/scripts/base/frameworks/input/sqlite/basic.bro

@@ -3,7 +3,7 @@
 #
 # @TEST-EXEC: cat conn.sql | sqlite3 conn.sqlite
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
 
 @TEST-START-FILE conn.sql