Merge remote-tracking branch 'origin/master' into topic/johanna/config-cluster

CHANGES

@@ -1,4 +1,54 @@
 
+2.5-747 | 2018-07-18 09:51:13 -0500
+
+  * Improve some netcontrol unit tests (Jon Siwek, Corelight)
+
+2.5-746 | 2018-07-17 17:51:13 -0500
+
+  * Improve an input framework unit test (Jon Siwek, Corelight)
+
+2.5-745 | 2018-07-17 16:46:16 -0500
+
+  * Add explicit key in Travis known_hosts (Jon Siwek, Corelight)
+
+2.5-743 | 2018-07-17 14:20:19 -0500
+
+  * Port broker::data variant usages to use CAF API directly
+    (Jon Siwek, Corelight)
+
+2.5-741 | 2018-07-16 16:06:02 -0500
+
+  * Improve Specific_RE_Matcher::CompileSet() error condition cleanup
+    (Jon Siwek, Corelight)
+
+2.5-740 | 2018-07-16 16:01:31 -0500
+
+  * Add support for case-insensitive patterns (Vern Paxson, Corelight)
+
+2.5-730 | 2018-07-16 10:39:33 -0500
+
+  * de-restrict pattern-oriented BiFs to no longer require only running at init
+    (Vern Paxson)
+
+  * Add option to toggle extraction of subject alternate names from X509 SAN
+    DNS field (Liviu Valsan)
+
+2.5-725 | 2018-07-03 14:56:10 -0500
+
+  * BIT-1941: improve unit test stability (Corelight)
+
+2.5-723 | 2018-07-03 09:34:10 -0500
+
+  * Fix unstable config framework test (Corelight)
+
+2.5-722 | 2018-07-03 09:16:37 -0500
+
+  * BIT-1941: teach diff-remove-timestamps about time 0 (Corelight)
+
+2.5-721 | 2018-07-02 16:29:21 -0500
+
+  * BIT-1941: improve reliability of broker.disconnect unit test (Corelight)
+
 2.5-719 | 2018-06-27 20:02:52 -0500
 
   * Fix some typos and formatting in NEWS and other documentation

NEWS

@@ -255,6 +255,21 @@ New Functionality
   semi-present in previous versions of Bro, but required constants as
   its operands; now you can use any pattern-valued expressions.
 
+- You can now specify that a pattern matches in a case-insensitive
+  fashion by adding 'i' to the end of its specification.  So for example
+  /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar".  Characters
+  enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar"
+  yields F, though it yields T for "xfOObar".
+
+  You can achieve the same functionality for a subpattern enclosed in
+  parentheses by adding "?i:" to the open parenthesis.  So for example
+  "/foo|(?i:bar)/" will match "BaR", but not "FoO".
+
+  For both ways of specifying case-insensitivity, characters enclosed in
+  double quotes maintain their case-sensitivity.  So for example /"foo"/i
+  will not match "Foo", but it will match "foo".
+
+
 Changed Functionality
 ---------------------
 
@@ -331,6 +346,9 @@ Changed Functionality
 
 - The "dnp3_header_block" event no longer has the "start" parameter
 
+- The string_to_pattern() built-in (and the now-deprecated merge_pattern()
+  built-in) is no longer restricted to only be called at initialization time.
+
 Removed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.5-719
+2.5-747

aux/bro-aux

@@ -1 +1 @@
-Subproject commit eeb677ff696f8ea3eaa43a765fe40da07ed5281d
+Subproject commit d2476564e6934de8fcffc47bff1ae9733c3dde0c

aux/broctl

@@ -1 +1 @@
-Subproject commit c5dd2ba83dda185d2008731a5cd25b2b8131ac78
+Subproject commit e28feb1ed80a5787f4d6055fbe15b1699397871e

aux/broker

@@ -1 +1 @@
-Subproject commit 1d90b931cc888e31b0d4774dbae54f5758841ab3
+Subproject commit 2285177616bf0e0bed6758bccf63f3dfee4d2b4f

doc/script-reference/types.rst

@@ -198,9 +198,9 @@ Here is a more detailed description of each type:
 
 .. bro:type:: pattern
 
-    A type representing regular-expression patterns which can be used
+    A type representing regular-expression patterns that can be used
     for fast text-searching operations.  Pattern constants are created
-    by enclosing text within forward slashes (/) and is the same syntax
+    by enclosing text within forward slashes (``/``) and use the same syntax
     as the patterns supported by the `flex lexical analyzer
     <http://westes.github.io/flex/manual/Patterns.html>`_.  The speed of
     regular expression matching does not depend on the complexity or
@@ -244,13 +244,25 @@ Here is a more detailed description of each type:
 
     yields true, like in the similar example above.  You can also
     create the conjunction (concatenation) of patterns using the ``&``
-    operator.  For example:
+    operator.  For example::
 
 	/foo/ & /bar/ in "foobar"
 
     will yield true because the pattern /(foo)(bar)/ appears in
     the string "foobar".
 
+    When specifying a pattern, you can add a final ``i`` specifier to
+    mark it as case-insensitive.  For example, ``/foo|bar/i`` will match
+    a "foo", "Foo", "BaR", etc.
+
+    You can also introduce a case-insensitive sub-pattern by enclosing it
+    in ``(?i:``<pattern>``)``.  So, for example, ``/foo|(?i:bar)/`` will
+    match "foo" and "BaR", but *not* "Foo".
+
+    For both ways of specifying case-insensitivity, characters enclosed
+    in double quotes maintain their case-sensitivity.  So for example
+    /"foo"/i will not match "Foo", but it will match "foo".
+
 .. bro:type:: port
 
     A type representing transport-level port numbers (besides TCP and

scripts/policy/frameworks/intel/seen/x509.bro

@@ -2,9 +2,16 @@
 @load base/files/x509
 @load ./where-locations
 
+module Intel;
+
+export {
+        ## Enables the extraction of subject alternate names from the X509 SAN DNS field
+        const enable_x509_ext_subject_alternative_name = T &redef;
+}
+
 event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName)
 	{
-	if ( ext?$dns )
+	if ( enable_x509_ext_subject_alternative_name && ext?$dns )
 		{
 		for ( i in ext$dns )
 			Intel::seen([$indicator=ext$dns[i],

src/3rdparty

@@ -1 +1 @@
-Subproject commit 648ff9aee1bb568a1a8252bd6e8146a7c60a911e
+Subproject commit b7c6be774b922be1e15f53571201c3be2bc28b75

src/RE.cc

@@ -102,6 +102,19 @@ void Specific_RE_Matcher::AddPat(const char* new_pat,
 	pattern_text = s;
 	}
 
+void Specific_RE_Matcher::MakeCaseInsensitive()
+	{
+	const char fmt[] = "(?i:%s)";
+	int n = strlen(pattern_text) + strlen(fmt);
+
+	char* s = new char[n + 5 /* slop */];
+
+	safe_snprintf(s, n + 5, fmt, pattern_text);
+
+	delete [] pattern_text;
+	pattern_text = s;
+	}
+
 int Specific_RE_Matcher::Compile(int lazy)
 	{
 	if ( ! pattern_text )
@@ -153,11 +166,12 @@ int Specific_RE_Matcher::CompileSet(const string_list& set, const int_list& idx)
 			{
 			reporter->Error("error compiling pattern /%s/", set[i]);
 
-			if ( set_nfa != nfa )
+			if ( set_nfa && set_nfa != nfa )
 				Unref(set_nfa);
-			Unref(nfa);
+			else
-			nfa = 0;
+				Unref(nfa);
 
+			nfa = 0;
 			return 0;
 			}
 
@@ -444,6 +458,12 @@ void RE_Matcher::AddPat(const char* new_pat)
 	re_exact->AddPat(new_pat);
 	}
 
+void RE_Matcher::MakeCaseInsensitive()
+	{
+	re_anywhere->MakeCaseInsensitive();
+	re_exact->MakeCaseInsensitive();
+	}
+
 int RE_Matcher::Compile(int lazy)
 	{
 	return re_anywhere->Compile(lazy) && re_exact->Compile(lazy);

src/RE.h

@@ -54,6 +54,8 @@ public:
 
 	void AddPat(const char* pat);
 
+	void MakeCaseInsensitive();
+
 	void SetPat(const char* pat)	{ pattern_text = copy_string(pat); }
 
 	int Compile(int lazy = 0);
@@ -178,6 +180,9 @@ public:
 
 	void AddPat(const char* pat);
 
+	// Makes the matcher as specified to date case-insensitive.
+	void MakeCaseInsensitive();
+
 	int Compile(int lazy = 0);
 
 	// Returns true if s exactly matches the pattern, false otherwise.

src/bro.bif

@@ -2960,12 +2960,6 @@ function uuid_to_string%(uuid: string%): string
 ##      :bro:id:`bro_init`.
 function merge_pattern%(p1: pattern, p2: pattern%): pattern &deprecated
 	%{
-	if ( bro_start_network_time != 0.0 )
-		{
-		builtin_error("merge_pattern can only be called at init time");
-		return 0;
-		}
-
 	RE_Matcher* re = new RE_Matcher();
 	re->AddPat(p1->PatternText());
 	re->AddPat(p2->PatternText());
@@ -3033,12 +3027,6 @@ function convert_for_pattern%(s: string%): string
 ##      :bro:id:`bro_init`.
 function string_to_pattern%(s: string, convert: bool%): pattern
 	%{
-	if ( bro_start_network_time != 0.0 )
-		{
-		builtin_error("string_to_pattern can only be called at init time");
-		return 0;
-		}
-
 	const char* ss = (const char*) (s->Bytes());
 	int sn = s->Len();
 	char* pat;

src/broker/Data.cc

@@ -215,7 +215,7 @@ struct val_converter {
 			{
 			auto expected_index_types = tt->Indices()->Types();
 			broker::vector composite_key;
-			auto indices = broker::get_if<broker::vector>(item);
+			auto indices = caf::get_if<broker::vector>(&item);
 
 			if ( indices )
 				{
@@ -283,7 +283,7 @@ struct val_converter {
 			{
 			auto expected_index_types = tt->Indices()->Types();
 			broker::vector composite_key;
-			auto indices = broker::get_if<broker::vector>(item.first);
+			auto indices = caf::get_if<broker::vector>(&item.first);
 
 			if ( indices )
 				{
@@ -384,7 +384,7 @@ struct val_converter {
 					return nullptr;
 					}
 
-				if ( broker::get_if<broker::none>(a[idx]) != nullptr )
+				if ( caf::get_if<broker::none>(&a[idx]) != nullptr )
 					{
 					rval->Assign(i, nullptr);
 					++idx;
@@ -411,8 +411,8 @@ struct val_converter {
 			if ( a.size() != 2 )
 				return nullptr;
 
-			auto exact_text = broker::get_if<std::string>(a[0]);
+			auto exact_text = caf::get_if<std::string>(&a[0]);
-			auto anywhere_text = broker::get_if<std::string>(a[1]);
+			auto anywhere_text = caf::get_if<std::string>(&a[1]);
 
 			if ( ! exact_text || ! anywhere_text )
 				return nullptr;
@@ -582,7 +582,7 @@ struct type_checker {
 		for ( const auto& item : a )
 			{
 			auto expected_index_types = tt->Indices()->Types();
-			auto indices = broker::get_if<broker::vector>(item);
+			auto indices = caf::get_if<broker::vector>(&item);
 			vector<const broker::data*> indices_to_check;
 
 			if ( indices )
@@ -624,7 +624,7 @@ struct type_checker {
 				auto expect = (*expected_index_types)[i];
 				auto& index_to_check = *(indices_to_check)[i];
 
-				if ( ! broker::visit(type_checker{expect}, index_to_check) )
+				if ( ! caf::visit(type_checker{expect}, index_to_check) )
 					return false;
 				}
 			}
@@ -642,7 +642,7 @@ struct type_checker {
 		for ( auto& item : a )
 			{
 			auto expected_index_types = tt->Indices()->Types();
-			auto indices = broker::get_if<broker::vector>(item.first);
+			auto indices = caf::get_if<broker::vector>(&item.first);
 			vector<const broker::data*> indices_to_check;
 
 			if ( indices )
@@ -689,11 +689,11 @@ struct type_checker {
 				auto expect = (*expected_index_types)[i];
 				auto& index_to_check = *(indices_to_check)[i];
 
-				if ( ! broker::visit(type_checker{expect}, index_to_check) )
+				if ( ! caf::visit(type_checker{expect}, index_to_check) )
 					return false;
 				}
 
-			if ( ! broker::visit(type_checker{tt->YieldType()},
+			if ( ! caf::visit(type_checker{tt->YieldType()},
 			                     item.second) )
 				return false;
 			}
@@ -709,7 +709,7 @@ struct type_checker {
 
 			for ( auto& item : a )
 				{
-				if ( ! broker::visit(type_checker{vt->YieldType()}, item) )
+				if ( ! caf::visit(type_checker{vt->YieldType()}, item) )
 					return false;
 				}
 
@@ -725,13 +725,13 @@ struct type_checker {
 				if ( idx >= a.size() )
 					return false;
 
-				if ( broker::get_if<broker::none>(a[idx]) != nullptr )
+				if ( caf::get_if<broker::none>(&a[idx]) != nullptr )
 					{
 					++idx;
 					continue;
 					}
 
-				if ( ! broker::visit(type_checker{rt->FieldType(i)},
+				if ( ! caf::visit(type_checker{rt->FieldType(i)},
 				                     a[idx]) )
 					return false;
 
@@ -745,8 +745,8 @@ struct type_checker {
 			if ( a.size() != 2 )
 				return false;
 
-			auto exact_text = broker::get_if<std::string>(a[0]);
+			auto exact_text = caf::get_if<std::string>(&a[0]);
-			auto anywhere_text = broker::get_if<std::string>(a[1]);
+			auto anywhere_text = caf::get_if<std::string>(&a[1]);
 
 			if ( ! exact_text || ! anywhere_text )
 				return false;
@@ -775,7 +775,7 @@ Val* bro_broker::data_to_val(broker::data d, BroType* type)
 	if ( type->Tag() == TYPE_ANY )
 		return bro_broker::make_data_val(move(d));
 
-	return broker::visit(val_converter{type}, std::move(d));
+	return caf::visit(val_converter{type}, std::move(d));
 	}
 
 broker::expected<broker::data> bro_broker::val_to_data(Val* v)
@@ -900,7 +900,7 @@ broker::expected<broker::data> bro_broker::val_to_data(Val* v)
 				key = move(composite_key);
 
 			if ( is_set )
-				broker::get<broker::set>(rval).emplace(move(key));
+				caf::get<broker::set>(rval).emplace(move(key));
 			else
 				{
 				auto val = val_to_data(entry->Value());
@@ -908,7 +908,7 @@ broker::expected<broker::data> bro_broker::val_to_data(Val* v)
 				if ( ! val )
 					return broker::ec::invalid_data;
 
-				broker::get<broker::table>(rval).emplace(move(key), move(*val));
+				caf::get<broker::table>(rval).emplace(move(key), move(*val));
 				}
 			}
 
@@ -1115,7 +1115,7 @@ struct data_type_getter {
 
 EnumVal* bro_broker::get_data_type(RecordVal* v, Frame* frame)
 	{
-	return broker::visit(data_type_getter{}, opaque_field_to_data(v, frame));
+	return caf::visit(data_type_getter{}, opaque_field_to_data(v, frame));
 	}
 
 broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f)
@@ -1131,7 +1131,7 @@ broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f)
 
 bool bro_broker::DataVal::canCastTo(BroType* t) const
 	{
-	return broker::visit(type_checker{t}, data);
+	return caf::visit(type_checker{t}, data);
 	}
 
 Val* bro_broker::DataVal::castTo(BroType* t)
@@ -1192,24 +1192,24 @@ broker::data bro_broker::threading_field_to_data(const threading::Field* f)
 
 threading::Field* bro_broker::data_to_threading_field(broker::data d)
 	{
-	if ( ! broker::is<broker::vector>(d) )
+	if ( ! caf::holds_alternative<broker::vector>(d) )
 		return nullptr;
 
-	auto& v = broker::get<broker::vector>(d);
+	auto& v = caf::get<broker::vector>(d);
-	auto name = broker::get_if<std::string>(v[0]);
+	auto name = caf::get_if<std::string>(&v[0]);
 	auto secondary = v[1];
-	auto type = broker::get_if<broker::count>(v[2]);
+	auto type = caf::get_if<broker::count>(&v[2]);
-	auto subtype = broker::get_if<broker::count>(v[3]);
+	auto subtype = caf::get_if<broker::count>(&v[3]);
-	auto optional = broker::get_if<broker::boolean>(v[4]);
+	auto optional = caf::get_if<broker::boolean>(&v[4]);
 
 	if ( ! (name && type && subtype && optional) )
 		return nullptr;
 
-	if ( secondary != broker::nil && ! broker::is<std::string>(secondary) )
+	if ( secondary != broker::nil && ! caf::holds_alternative<std::string>(secondary) )
 		return nullptr;
 
 	return new threading::Field(name->c_str(),
-				    secondary != broker::nil ? broker::get<std::string>(secondary).c_str() : nullptr,
+				    secondary != broker::nil ? caf::get<std::string>(secondary).c_str() : nullptr,
 				    static_cast<TypeTag>(*type),
 				    static_cast<TypeTag>(*subtype),
 				    *optional);

src/broker/Data.h

@@ -210,11 +210,11 @@ broker::data& opaque_field_to_data(RecordVal* v, Frame* f);
 template <typename T>
 T& require_data_type(broker::data& d, TypeTag tag, Frame* f)
 	{
-	auto ptr = broker::get_if<T>(d);
+	auto ptr = caf::get_if<T>(&d);
 	if ( ! ptr )
 		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
 		                       "data is of type '%s' not of type '%s'",
-		                       broker::visit(type_name_getter{tag}, d),
+		                       caf::visit(type_name_getter{tag}, d),
 		                       type_name(tag));
 
 	return *ptr;

src/broker/Manager.cc

@@ -942,13 +942,13 @@ void Manager::Process()
 		{
 		had_input = true;
 
-		if ( auto stat = broker::get_if<broker::status>(status_msg) )
+		if ( auto stat = caf::get_if<broker::status>(&status_msg) )
 			{
 			ProcessStatus(std::move(*stat));
 			continue;
 			}
 
-		if ( auto err = broker::get_if<broker::error>(status_msg) )
+		if ( auto err = caf::get_if<broker::error>(&status_msg) )
 			{
 			ProcessError(std::move(*err));
 			continue;
@@ -1048,7 +1048,7 @@ void Manager::ProcessRelayEvent(broker::bro::RelayEvent ev)
 	++statistics.num_events_incoming;
 
 	for ( auto& t : ev.topics() )
-		PublishEvent(std::move(broker::get<std::string>(t)),
+		PublishEvent(std::move(caf::get<std::string>(t)),
 		             std::move(ev.name()),
 		             std::move(ev.args()));
 	}
@@ -1060,7 +1060,7 @@ void Manager::ProcessHandleAndRelayEvent(broker::bro::HandleAndRelayEvent ev)
 	ProcessEvent(ev.name(), ev.args());
 
 	for ( auto& t : ev.topics() )
-		PublishEvent(std::move(broker::get<std::string>(t)),
+		PublishEvent(std::move(caf::get<std::string>(t)),
 		             std::move(ev.name()),
 		             std::move(ev.args()));
 	}
@@ -1095,40 +1095,38 @@ bool bro_broker::Manager::ProcessLogCreate(broker::bro::LogCreate lc)
 		}
 
 	// Get log fields.
+	auto fields_data = caf::get_if<broker::vector>(&lc.fields_data());
 
-	try
+	if ( ! fields_data )
-		{
-		auto fields_data = std::move(broker::get<broker::vector>(lc.fields_data()));
-		auto num_fields = fields_data.size();
-		auto fields = new threading::Field* [num_fields];
-
-		for ( auto i = 0u; i < num_fields; ++i )
-			{
-			if ( auto field = data_to_threading_field(std::move(fields_data[i])) )
-				fields[i] = field;
-			else
-				{
-				reporter->Warning("failed to convert remote log field # %d", i);
-				return false;
-				}
-			}
-
-		if ( ! log_mgr->CreateWriterForRemoteLog(stream_id->AsEnumVal(), writer_id->AsEnumVal(), writer_info.get(), num_fields, fields) )
-			{
-			ODesc d;
-			stream_id->Describe(&d);
-			reporter->Warning("failed to create remote log stream for %s locally", d.Description());
-			}
-
-		writer_info.release(); // log_mgr took ownership.
-		return true;
-		}
-
-	catch (const broker::bad_variant_access& e)
 		{
 		reporter->Warning("failed to unpack remote log fields");
 		return false;
 		}
+
+	auto num_fields = fields_data->size();
+	auto fields = new threading::Field* [num_fields];
+
+	for ( auto i = 0u; i < num_fields; ++i )
+		{
+		if ( auto field = data_to_threading_field(std::move((*fields_data)[i])) )
+			fields[i] = field;
+		else
+			{
+			reporter->Warning("failed to convert remote log field # %d", i);
+			delete [] fields;
+			return false;
+			}
+		}
+
+	if ( ! log_mgr->CreateWriterForRemoteLog(stream_id->AsEnumVal(), writer_id->AsEnumVal(), writer_info.get(), num_fields, fields) )
+		{
+		ODesc d;
+		stream_id->Describe(&d);
+		reporter->Warning("failed to create remote log stream for %s locally", d.Description());
+		}
+
+	writer_info.release(); // log_mgr took ownership.
+	return true;
 	}
 
 bool bro_broker::Manager::ProcessLogWrite(broker::bro::LogWrite lw)
@@ -1159,52 +1157,56 @@ bool bro_broker::Manager::ProcessLogWrite(broker::bro::LogWrite lw)
 		}
 
 	unref_guard writer_id_unreffer{writer_id};
+	auto path = caf::get_if<std::string>(&lw.path());
 
-	 try
+	if ( ! path )
 		{
-		auto& path = broker::get<std::string>(lw.path());
+		reporter->Warning("failed to unpack remote log values (bad path variant) for stream: %s", stream_id_name.data());
-		auto& serial_data = broker::get<std::string>(lw.serial_data());
-
-		BinarySerializationFormat fmt;
-		fmt.StartRead(serial_data.data(), serial_data.size());
-
-		int num_fields;
-		bool success = fmt.Read(&num_fields, "num_fields");
-
-		if ( ! success )
-			{
-			reporter->Warning("failed to unserialize remote log num fields for stream: %s", stream_id_name.data());
-			return false;
-			}
-
-		auto vals = new threading::Value* [num_fields];
-
-		for ( int i = 0; i < num_fields; ++i )
-			{
-			vals[i] = new threading::Value;
-
-			if ( ! vals[i]->Read(&fmt) )
-				{
-				for ( int j = 0; j <=i; ++j )
-					delete vals[j];
-
-				delete [] vals;
-				reporter->Warning("failed to unserialize remote log field %d for stream: %s", i, stream_id_name.data());
-
-				return false;
-				}
-			}
-
-		log_mgr->WriteFromRemote(stream_id->AsEnumVal(), writer_id->AsEnumVal(), std::move(path), num_fields, vals);
-		fmt.EndRead();
-		return true;
-		}
-
-	catch ( const broker::bad_variant_access& e)
-		{
-		reporter->Warning("failed to unpack remote log values (bad variant) for stream: %s", stream_id_name.data());
 		return false;
 		}
+
+	auto serial_data = caf::get_if<std::string>(&lw.serial_data());
+
+	if ( ! serial_data )
+		{
+		reporter->Warning("failed to unpack remote log values (bad serial_data variant) for stream: %s", stream_id_name.data());
+		return false;
+		}
+
+	BinarySerializationFormat fmt;
+	fmt.StartRead(serial_data->data(), serial_data->size());
+
+	int num_fields;
+	bool success = fmt.Read(&num_fields, "num_fields");
+
+	if ( ! success )
+		{
+		reporter->Warning("failed to unserialize remote log num fields for stream: %s", stream_id_name.data());
+		return false;
+		}
+
+	auto vals = new threading::Value* [num_fields];
+
+	for ( int i = 0; i < num_fields; ++i )
+		{
+		vals[i] = new threading::Value;
+
+		if ( ! vals[i]->Read(&fmt) )
+			{
+			for ( int j = 0; j <=i; ++j )
+				delete vals[j];
+
+			delete [] vals;
+			reporter->Warning("failed to unserialize remote log field %d for stream: %s", i, stream_id_name.data());
+
+			return false;
+			}
+		}
+
+	log_mgr->WriteFromRemote(stream_id->AsEnumVal(), writer_id->AsEnumVal(),
+	                         std::move(*path), num_fields, vals);
+	fmt.EndRead();
+	return true;
 	}
 
 bool Manager::ProcessIdentifierUpdate(broker::bro::IdentifierUpdate iu)

src/broker/data.bif

@@ -457,7 +457,7 @@ function Broker::__record_lookup%(r: Broker::Data, idx: count%): Broker::Data
 	auto& v = bro_broker::require_data_type<broker::vector>(r->AsRecordVal(),
 	                                                        TYPE_RECORD, frame);
 
-	if ( idx >= v.size() || broker::get_if<broker::none>(v[idx]) )
+	if ( idx >= v.size() || caf::get_if<broker::none>(&v[idx]) )
 		return new RecordVal(BifType::Record::Broker::Data);
 
 	return bro_broker::make_data_val(v[idx]);
@@ -496,7 +496,7 @@ function Broker::__record_iterator_value%(it: opaque of Broker::RecordIterator%)
 		return rval;
 		}
 
-	if ( broker::get_if<broker::none>(*ri->it) )
+	if ( caf::get_if<broker::none>(&(*ri->it)) )
 		return rval; // field isn't set
 
 	rval->Assign(0, new bro_broker::DataVal(*ri->it));

src/input.h

@@ -23,7 +23,6 @@ extern void add_input_file_at_front(const char* file);
 extern void add_to_name_list(char* s, char delim, name_list& nl);
 
 extern void begin_RE();
-extern void end_RE();
 
 extern void do_atif(Expr* expr);
 extern void do_atifdef(const char* id);

src/logging/WriterBackend.cc

@@ -137,15 +137,15 @@ broker::data WriterBackend::WriterInfo::ToBroker() const
 
 bool WriterBackend::WriterInfo::FromBroker(broker::data d)
 	{
-	if ( ! broker::is<broker::vector>(d) )
+	if ( ! caf::holds_alternative<broker::vector>(d) )
 		return false;
 
-	auto v = broker::get<broker::vector>(d);
+	auto v = caf::get<broker::vector>(d);
-	auto bpath = broker::get_if<std::string>(v[0]);
+	auto bpath = caf::get_if<std::string>(&v[0]);
-	auto brotation_base = broker::get_if<double>(v[1]);
+	auto brotation_base = caf::get_if<double>(&v[1]);
-	auto brotation_interval = broker::get_if<double>(v[2]);
+	auto brotation_interval = caf::get_if<double>(&v[2]);
-	auto bnetwork_time = broker::get_if<double>(v[3]);
+	auto bnetwork_time = caf::get_if<double>(&v[3]);
-	auto bconfig = broker::get_if<broker::table>(v[4]);
+	auto bconfig = caf::get_if<broker::table>(&v[4]);
 
 	if ( ! (bpath && brotation_base && brotation_interval && bnetwork_time && bconfig) )
 		return false;
@@ -157,8 +157,8 @@ bool WriterBackend::WriterInfo::FromBroker(broker::data d)
 
 	for ( auto i : *bconfig )
 		{
-		auto k = broker::get_if<std::string>(i.first);
+		auto k = caf::get_if<std::string>(&i.first);
-		auto v = broker::get_if<std::string>(i.second);
+		auto v = caf::get_if<std::string>(&i.second);
 
 		if ( ! (k && v) )
 			return false;

src/parse.y

@@ -14,7 +14,7 @@
 %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FALLTHROUGH
 %token TOK_FILE TOK_FOR TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT
 %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE
-%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT
+%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_END TOK_PATTERN_TEXT
 %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF
 %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET
 %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE
@@ -52,7 +52,7 @@
 %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR
 %nonassoc TOK_AS TOK_IS
 
-%type <b> opt_no_test opt_no_test_block opt_deprecated
+%type <b> opt_no_test opt_no_test_block opt_deprecated TOK_PATTERN_END
 %type <str> TOK_ID TOK_PATTERN_TEXT
 %type <id> local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func case_type
 %type <id_l> local_id_list case_type_list
@@ -723,13 +723,16 @@ expr:
 			$$ = new ConstExpr($1);
 			}
 
-	|	'/' { begin_RE(); } TOK_PATTERN_TEXT { end_RE(); } '/'
+	|	'/' { begin_RE(); } TOK_PATTERN_TEXT TOK_PATTERN_END
 			{
 			set_location(@3);
 
 			RE_Matcher* re = new RE_Matcher($3);
 			delete [] $3;
 
+			if ( $4 )
+				re->MakeCaseInsensitive();
+
 			re->Compile();
 			$$ = new ConstExpr(new PatternVal(re));
 			}

src/re-parse.y

@@ -11,11 +11,12 @@
 int csize = 256;
 int syntax_error = 0;
 
+int cupper(int sym);
 int clower(int sym);
 void yyerror(const char msg[]);
 %}
 
-%token TOK_CHAR TOK_NUMBER TOK_CCL TOK_CCE
+%token TOK_CHAR TOK_NUMBER TOK_CCL TOK_CCE TOK_CASE_INSENSITIVE
 
 %union {
 	int int_val;
@@ -126,12 +127,11 @@ singleton	:  singleton '*'
 		|  '(' re ')'
 			{ $$ = $2; }
 
+		|  TOK_CASE_INSENSITIVE re ')'
+			{ $$ = $2; case_insensitive = 0; }
+
 		|  TOK_CHAR
-			{
+			{ $$ = new NFA_Machine(new NFA_State($1, rem->EC())); }
-			if ( case_insensitive && $1 >= 'A' && $1 <= 'Z' )
-				$1 = clower($1);
-			$$ = new NFA_Machine(new NFA_State($1, rem->EC()));
-			}
 
 		|  '^'
 			{
@@ -158,17 +158,29 @@ full_ccl	:  '[' ccl ']'
 
 ccl		:  ccl TOK_CHAR '-' TOK_CHAR
 			{
-			if ( case_insensitive )
-				{
-				if ( $2 >= 'A' && $2 <= 'Z' )
-					$2 = clower($2);
-				if ( $4 >= 'A' && $4 <= 'Z' )
-					$4 = clower($4);
-				}
-
 			if ( $2 > $4 )
 				synerr("negative range in character class");
 
+			else if ( case_insensitive &&
+				  (isalpha($2) || isalpha($4)) )
+				{
+				if ( isalpha($2) && isalpha($4) &&
+				     isupper($2) == isupper($4) )
+					{ // Compatible range, do both versions
+					int l2 = tolower($2);
+					int l4 = tolower($4);
+
+					for ( int i = l2; i<= l4; ++i )
+						{
+						$1->Add(i);
+						$1->Add(toupper(i));
+						}
+					}
+
+				else
+					synerr("ambiguous case-insensitive character class");
+				}
+
 			else
 				{
 				for ( int i = $2; i <= $4; ++i )
@@ -178,10 +190,13 @@ ccl		:  ccl TOK_CHAR '-' TOK_CHAR
 
 		|  ccl TOK_CHAR
 			{
-			if ( case_insensitive && $2 >= 'A' && $2 <= 'Z' )
+			if ( case_insensitive && isalpha($2) )
-				$2 = clower($2);
+				{
-
+				$1->Add(clower($2));
-			$1->Add($2);
+				$1->Add(cupper($2));
+				}
+			else
+				$1->Add($2);
 			}
 
 		|  ccl ccl_expr
@@ -200,9 +215,10 @@ ccl_expr:	   TOK_CCE
 
 string		:  string TOK_CHAR
 			{
-			if ( case_insensitive && $2 >= 'A' && $2 <= 'Z' )
+			// Even if case-insensitivity is set,
-				$2 = clower($2);
+			// leave this alone; that provides a way
-
+			// of "escaping" out of insensitivity
+			// if needed.
 			$1->AppendState(new NFA_State($2, rem->EC()));
 			}
 
@@ -211,6 +227,11 @@ string		:  string TOK_CHAR
 		;
 %%
 
+int cupper(int sym)
+	{
+	return (isascii(sym) && islower(sym)) ?  toupper(sym) : sym;
+	}
+
 int clower(int sym)
 	{
 	return (isascii(sym) && isupper(sym)) ?  tolower(sym) : sym;

src/re-scan.l

@@ -114,6 +114,25 @@ CCL_EXPR	("[:"[[:alpha:]]+":]")
 				}
 			}
 
+	"(?i:"		case_insensitive = 1; return TOK_CASE_INSENSITIVE;
+
+	[a-zA-Z]	{
+			if ( case_insensitive )
+				{
+				char c = yytext[0];	// unput trashes yytext!
+				// Push back the character inside a CCL,
+				// so the parser can then expand it.
+				unput(']');
+				unput(c);
+				unput('[');
+				}
+			else
+				{
+				yylval.int_val = yytext[0];
+				return TOK_CHAR;
+				}
+			}
+
 	[|*+?.(){}]	return yytext[0];
 	.		yylval.int_val = yytext[0]; return TOK_CHAR;
 	\n		return 0;	// treat as end of pattern
@@ -149,15 +168,22 @@ CCL_EXPR	("[:"[[:alpha:]]+":]")
 	"[:cntrl:]"	RET_CCE(my_iscntrl)
 	"[:digit:]"	RET_CCE(my_isdigit)
 	"[:graph:]"	RET_CCE(my_isgraph)
-	"[:lower:]"	RET_CCE(my_islower)
 	"[:print:]"	RET_CCE(my_isprint)
 	"[:punct:]"	RET_CCE(my_ispunct)
 	"[:space:]"	RET_CCE(my_isspace)
 	"[:xdigit:]"	RET_CCE(my_isxdigit)
+
+	"[:lower:]"	{
+			BEGIN(SC_CCL);
+			yylval.cce_val =
+				case_insensitive ? my_isalpha : my_islower;
+			return TOK_CCE;
+			}
+
 	"[:upper:]"	{
 			BEGIN(SC_CCL);
 			yylval.cce_val =
-				case_insensitive ? my_isupper : my_islower;
+				case_insensitive ? my_isalpha : my_isupper;
 			return TOK_CCE;
 			}
 

src/scan.l

@@ -554,7 +554,19 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 	return TOK_PATTERN_TEXT;
 	}
 
-<RE>[/\\\n]	return yytext[0];
+<RE>"/"	{
+	BEGIN(INITIAL);
+	yylval.b = false;
+	return TOK_PATTERN_END;
+	}
+
+<RE>"/i"	{
+	BEGIN(INITIAL);
+	yylval.b = true;
+	return TOK_PATTERN_END;
+	}
+
+<RE>[\\\n]	return yytext[0];	// should cause a parse error
 
 <*>.	reporter->Error("unrecognized character - %s", yytext);
 
@@ -698,11 +710,6 @@ void begin_RE()
 	BEGIN(RE);
 	}
 
-void end_RE()
-	{
-	BEGIN(INITIAL);
-	}
-
 class LocalNameFinder : public TraversalCallback {
 public:
 	LocalNameFinder()

testing/btest/Baseline/broker.disconnect/recv.broker.filtered.log

@@ -1 +0,0 @@
-1502645128.235998 Broker::STATUS peer-added 127.0.0.1 XXX handshake successful

testing/btest/Baseline/broker.disconnect/recv.recv.out

@@ -1,2 +1,2 @@
 peer added, handshake successful
-Something receiver, 1
+receiver got event, 1

testing/btest/Baseline/broker.disconnect/recv2.broker.filtered.log

@@ -1 +0,0 @@
-1502645286.464675 Broker::STATUS peer-added 127.0.0.1 XXX handshake successful

testing/btest/Baseline/broker.disconnect/recv2.recv2.out

@@ -1,2 +1,2 @@
 peer added, handshake successful
-Something receiver, 2
+receiver got event, 2

testing/btest/Baseline/broker.disconnect/send.broker.filtered.log

@@ -1,4 +0,0 @@
-1524513026.795171 Broker::STATUS peer-added 127.0.0.1 XXX received handshake from remote core
-1524513033.340316 Broker::STATUS connection-terminated 127.0.0.1 XXX lost remote peer
-1524513035.437373 Broker::STATUS peer-added 127.0.0.1 XXX received handshake from remote core
-1524513041.743002 Broker::STATUS connection-terminated 127.0.0.1 XXX lost remote peer

testing/btest/Baseline/broker.disconnect/send.send.out

@@ -1,6 +1,4 @@
 peer added, received handshake from remote core
-Something sender, 1
 peer lost, lost remote peer
 peer added, received handshake from remote core
-Something sender, 2
 peer lost, lost remote peer

testing/btest/Baseline/language.pattern/out

@@ -10,3 +10,25 @@ in operator (PASS)
 & operator (FAIL)
 | operator (PASS)
 | operator (FAIL)
+/i pattern modifier (PASS)
+/i pattern modifier (PASS)
+/i double-quote escape (FAIL)
+/i double-quote escape (PASS)
+case-sensitive pattern (FAIL)
+case-sensitive pattern (FAIL)
+case-sensitive pattern (PASS)
+/i pattern disjunction (PASS)
+/i pattern disjunction (FAIL)
+/i pattern disjunction (PASS)
+/i pattern disjunction (PASS)
+/i pattern concatenation (PASS)
+/i pattern concatenation (FAIL)
+/i pattern concatenation (FAIL)
+/i pattern concatenation (PASS)
+/i pattern concatenation (PASS)
+/i pattern concatenation (FAIL)
+/i pattern character class (FAIL)
+/i pattern character class (PASS)
+(?i:...) pattern construct (PASS)
+(?i:...) pattern construct (FAIL)
+(?i:...) pattern construct (PASS)

testing/btest/Baseline/scripts.base.frameworks.input.raw.executestdin/out

@@ -1,20 +1,10 @@
-Input::EVENT_NEW, cat |, input0
+Input::EVENT_NEW, cat |, input0, hello
-hello
+Input::EVENT_NEW, cat |, input0, there\x01\x02\x03\x04\x05\x01\x02\x03yay0
-Input::EVENT_NEW, cat |, input0
+Input::EVENT_NEW, cat |, input1, hello
-there\x01\x02\x03\x04\x05\x01\x02\x03yay0
+Input::EVENT_NEW, cat |, input1, there\x01\x02\x03\x04\x05\x01\x02\x03yay01
-Input::EVENT_NEW, cat |, input1
+Input::EVENT_NEW, cat |, input4, hello
-hello
+Input::EVENT_NEW, cat |, input4, there\x01\x02\x03\x04\x05\x01\x02\x03yay01234
-Input::EVENT_NEW, cat |, input1
+Input::EVENT_NEW, cat |, input2, hello
-there\x01\x02\x03\x04\x05\x01\x02\x03yay01
+Input::EVENT_NEW, cat |, input2, there\x01\x02\x03\x04\x05\x01\x02\x03yay012
-Input::EVENT_NEW, cat |, input2
+Input::EVENT_NEW, cat |, input3, hello
-hello
+Input::EVENT_NEW, cat |, input3, there\x01\x02\x03\x04\x05\x01\x02\x03yay0123
-Input::EVENT_NEW, cat |, input2
-there\x01\x02\x03\x04\x05\x01\x02\x03yay012
-Input::EVENT_NEW, cat |, input3
-hello
-Input::EVENT_NEW, cat |, input3
-there\x01\x02\x03\x04\x05\x01\x02\x03yay0123
-Input::EVENT_NEW, cat |, input4
-hello
-Input::EVENT_NEW, cat |, input4
-there\x01\x02\x03\x04\x05\x01\x02\x03yay01234

testing/btest/Baseline/scripts.base.frameworks.input.raw.executestream/out

@@ -1,153 +1,25 @@
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 q3r3057fdf
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 sdfs\d
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 dfsdf
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 sdf
-[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+tail -f ../input.log |, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 3rw43wRRERLlL#RWERERERE.
 done

testing/btest/Baseline/scripts.base.frameworks.input.raw.stderr/out

@@ -1,27 +1,9 @@
-Input::EVENT_NEW
+Input::EVENT_NEW line output (stderr=F): ../mydir:
-..:
+Input::EVENT_NEW line output (stderr=F): a
-F
+Input::EVENT_NEW line output (stderr=F): b
-Input::EVENT_NEW
+Input::EVENT_NEW line output (stderr=F): c
-bro
+Input::EVENT_NEW line output (stderr=T): <stderr output contained nonexistant>
-F
+Input::EVENT_NEW line output (stderr=T): <stderr output contained nonexistant>
-Input::EVENT_NEW
+Input::EVENT_NEW line output (stderr=T): <stderr output contained nonexistant>
-out
+End of Data event, input
-F
+Process finished event, input, T
-Input::EVENT_NEW
-stderr.bro
-F
-Input::EVENT_NEW
-stderr output contained nonexistant
-T
-Input::EVENT_NEW
-stderr output contained nonexistant
-T
-Input::EVENT_NEW
-stderr output contained nonexistant
-T
-done
-End of Data event
-input
-Process finished event
-input
-Exit code != 0

testing/btest/Baseline/scripts.base.frameworks.input.raw.streamraw/out

@@ -1,153 +1,25 @@
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 q3r3057fdf
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 sdfs\d
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 dfsdf
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 sdf
-[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=<no value description>, want_record=F, ev=line
+../input.log, Input::READER_RAW, Input::STREAM, input
-{ 
-print A::outfile, A::description;
-print A::outfile, A::tpe;
-print A::outfile, A::s;
-A::try = A::try + 1;
-if (8 == A::try) 
-{ 
-print A::outfile, done;
-close(A::outfile);
-Input::remove(input);
-terminate();
-}
-
-}, error_ev=<uninitialized>, config={
-
-}]
 Input::EVENT_NEW
 3rw43wRRERLlL#RWERERERE.
 done

testing/btest/Baseline/scripts.base.frameworks.netcontrol.basic-cluster/worker-2..stdout

@@ -1,3 +1,4 @@
 Rule added, worker-2:2, 4
 Rule added, worker-2:3, 5
 1
+Rule destroyed, worker-2:3, 5, 0

testing/btest/broker/disconnect.bro

@@ -1,22 +1,19 @@
 # @TEST-SERIALIZE: comm
-#
+
 # @TEST-EXEC: btest-bg-run recv "bro -B broker -b ../recv.bro >recv.out"
 # @TEST-EXEC: btest-bg-run send "bro -B broker -b ../send.bro >send.out"
-#
+
-# @TEST-EXEC: sleep 6 && kill $(cat recv/.pid) && sleep 1 && echo 0 >recv/.exitcode
+# @TEST-EXEC: $SCRIPTS/wait-for-file recv/got-event 30 || (btest-bg-wait -k 1 && false)
+# @TEST-EXEC: kill $(cat recv/.pid)
+# @TEST-EXEC: $SCRIPTS/wait-for-pid $(cat recv/.pid) 10 || (btest-bg-wait -k 1 && false)
+# @TEST-EXEC: echo 0 >recv/.exitcode
+
 # @TEST-EXEC: btest-bg-run recv2 "bro -B broker -b ../recv.bro >recv2.out"
-#
+# @TEST-EXEC: btest-bg-wait 30
-# @TEST-EXEC: btest-bg-wait 25
+
 # @TEST-EXEC: btest-diff send/send.out
 # @TEST-EXEC: btest-diff recv/recv.out
 # @TEST-EXEC: btest-diff recv2/recv2.out
-#
-# @TEST-EXEC: cat send/broker.log | awk '/Broker::STATUS/ { $5="XXX"; print; }' >send/broker.filtered.log
-# @TEST-EXEC: cat recv/broker.log | awk '/Broker::STATUS/ { $5="XXX"; print; }' >recv/broker.filtered.log
-# @TEST-EXEC: cat recv2/broker.log | grep -v "lost remote peer" | awk '/Broker::STATUS/ { $5="XXX"; print; }' >recv2/broker.filtered.log
-# @TEST-EXEC: btest-diff send/broker.filtered.log
-# @TEST-EXEC: btest-diff recv/broker.filtered.log
-# @TEST-EXEC: btest-diff recv2/broker.filtered.log
 
 @TEST-START-FILE send.bro
 
@@ -24,42 +21,34 @@ redef Broker::default_connect_retry=1secs;
 redef Broker::default_listen_retry=1secs;
 redef exit_only_after_terminate = T;
 
-event self_terminate()
+global peers = 0;
+const test_topic = "bro/test/my_topic";
+
+event my_event(i: count)
 	{
-	terminate();
+	print "sender got event", i;
 	}
 
-event do_terminate()
-    {
-    schedule 2sec { self_terminate() };
-    }
-
-event print_something(i: int)
-    {
-    print "Something sender", i;
-    }
-
 event bro_init()
-    {
+	{
-    Broker::subscribe("bro/event/my_topic");
+	Broker::subscribe(test_topic);
-    Broker::auto_publish("bro/event/my_topic", print_something);
+	Broker::peer("127.0.0.1");
-    Broker::auto_publish("bro/event/my_topic", do_terminate);
+	}
-    Broker::peer("127.0.0.1");
-
-    schedule 3secs { print_something(1) };
-    schedule 12secs { print_something(2) };
-    schedule 13secs { do_terminate() };
-    }
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
-    {
+	{
-    print "peer lost", msg;
+	print "peer lost", msg;
-    }
+
+	if ( peers == 2 )
+		terminate();
+	}
 
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
-    {
+	{
-    print "peer added", msg;
+	++peers;
-    }
+	print "peer added", msg;
+	Broker::publish(test_topic, my_event, peers);
+	}
 
 @TEST-END-FILE
 
@@ -70,31 +59,33 @@ redef Broker::default_connect_retry=1secs;
 redef Broker::default_listen_retry=1secs;
 redef exit_only_after_terminate = T;
 
-event do_terminate()
+const test_topic = "bro/test/my_topic";
-    {
-    terminate();
-    }
 
-event print_something(i: int)
+event my_event(i: count)
-    {
+	{
-    print "Something receiver", i;
+	print "receiver got event", i;
-    }
+
+	if ( i == 1 )
+		# In the first case, terminate via `kill` from btest command.
+		system("touch got-event");
+	else
+		terminate();
+	}
 
 event bro_init()
-    {
+	{
-    Broker::subscribe("bro/event/my_topic");
+	Broker::subscribe(test_topic);
-    Broker::listen("127.0.0.1");
+	Broker::listen("127.0.0.1");
-    }
+	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
-    {
+	{
-    # In the 2nd run, this may be lost at termination, so don't output.
+	terminate();
-    #print "peer lost", msg;
+	}
-    }
 
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
-    {
+	{
-    print "peer added", msg;
+	print "peer added", msg;
-    }
+	}
 
 @TEST-END-FILE

testing/btest/core/leaks/basic-cluster.bro

@@ -6,7 +6,6 @@
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m %INPUT
 # @TEST-EXEC: btest-bg-wait 60
@@ -19,6 +18,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 global n = 0;

testing/btest/core/leaks/hll_cluster.bro

@@ -7,7 +7,6 @@
 #
 # @TEST-EXEC: bro -m %INPUT>out
 # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
-# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m runnumber=1 %INPUT
 # @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m runnumber=2 %INPUT
 # @TEST-EXEC: btest-bg-wait 60
@@ -24,6 +23,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 global hll_data: event(data: opaque of cardinality);

testing/btest/core/leaks/input-raw.bro

@@ -6,9 +6,9 @@
 #
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT 
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 8 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input2.log >> input.log
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 8 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input3.log >> input.log
 # @TEST-EXEC: btest-bg-wait 60
 
@@ -48,7 +48,12 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	print outfile, s;
 
 	try = try + 1;
-	if ( try == 16 )
+
+	if ( try == 2 )
+		system("touch got2");
+	else if ( try == 6 )
+		system("touch got6");
+	else if ( try == 16 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/core/leaks/input-reread.bro

@@ -6,13 +6,13 @@
 #
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT
-# @TEST-EXEC: sleep 60
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 60 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input2.log input.log
-# @TEST-EXEC: sleep 10
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 10 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input3.log input.log
-# @TEST-EXEC: sleep 10
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 10 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input4.log input.log
-# @TEST-EXEC: sleep 10
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got8 10 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input5.log input.log
 # @TEST-EXEC: btest-bg-wait 120
 
@@ -145,7 +145,16 @@ event Input::end_of_data(name: string, source: string)
 	}
 	
 	try = try + 1;
-	if ( try == 10 )
+
+	if ( try == 2 )
+		system("touch got2");
+	else if ( try == 4 )
+		system("touch got4");
+	else if ( try == 6 )
+		system("touch got6");
+	else if ( try == 8 )
+		system("touch got8");
+	else if ( try == 10 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/core/leaks/pattern.bro

@@ -35,4 +35,33 @@ event new_connection(c: connection)
 	test_case( "& operator", p2 & p1 in "baroob" );
 	test_case( "| operator", p1 | p2 in "lazybarlazy" );
 	test_case( "| operator", p3 | p4 in "xoob" );
+
+	test_case( "/i pattern modifier", /fOO/i in "xFoObar" );
+	test_case( "/i pattern modifier", /fOO/i == "Foo" );
+
+	test_case( "/i double-quote escape", /"fOO"/i in "xFoObar" );
+	test_case( "/i double-quote escape", /"fOO"/i in "xfOObar" );
+
+	test_case( "case-sensitive pattern", /fOO/ in "xFoObar" );
+	test_case( "case-sensitive pattern", /fOO/ == "Foo" );
+	test_case( "case-sensitive pattern", /fOO/ == "fOO" );
+
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bez" );
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bEz" );
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bar" );
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bAr" );
+
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "barbez" );
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "barbEz" );
+	test_case( "/i pattern concatenation", /BAR/i & /bez/ == "barbEz" );
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "bArbez" );
+	test_case( "/i pattern concatenation", /BAR/i & /bez/ == "bArbez" );
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "bArbEz" );
+
+	test_case( "/i pattern character class", /ba[0a-c99S-Z0]/i & /bEz/ == "bArbEz" );
+	test_case( "/i pattern character class", /ba[0a-c99M-S0]/i & /bEz/ == "bArbEz" );
+
+	test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ in "xBAry" );
+	test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ in "xFOoy" );
+	test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ | /foo/i in "xFOoy" );
 	}

testing/btest/language/pattern.bro

@@ -22,15 +22,47 @@ event bro_init()
 
 	test_case( "equality operator", "foo" == p1 );
 	test_case( "equality operator (order of operands)", p1 == "foo" );
+
 	test_case( "inequality operator", "foobar" != p1 );
 	test_case( "inequality operator (order of operands)", p1 != "foobar" );
+
 	test_case( "in operator", p1 in "foobar" );
 	test_case( "in operator", p2 in "foobar" );
 	test_case( "!in operator", p3 !in "foobar" );
+
 	test_case( "& operator", p1 & p2 in "baroob" );
 	test_case( "& operator", p2 & p1 in "baroob" );
+
 	test_case( "| operator", p1 | p2 in "lazybarlazy" );
 	test_case( "| operator", p3 | p4 in "xoob" );
 
-}
+	test_case( "/i pattern modifier", /fOO/i in "xFoObar" );
+	test_case( "/i pattern modifier", /fOO/i == "Foo" );
 
+	test_case( "/i double-quote escape", /"fOO"/i in "xFoObar" );
+	test_case( "/i double-quote escape", /"fOO"/i in "xfOObar" );
+
+	test_case( "case-sensitive pattern", /fOO/ in "xFoObar" );
+	test_case( "case-sensitive pattern", /fOO/ == "Foo" );
+	test_case( "case-sensitive pattern", /fOO/ == "fOO" );
+
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bez" );
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bEz" );
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bar" );
+	test_case( "/i pattern disjunction", /bar/i | /bez/ == "bAr" );
+
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "barbez" );
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "barbEz" );
+	test_case( "/i pattern concatenation", /BAR/i & /bez/ == "barbEz" );
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "bArbez" );
+	test_case( "/i pattern concatenation", /BAR/i & /bez/ == "bArbez" );
+	test_case( "/i pattern concatenation", /bar/i & /bez/ == "bArbEz" );
+
+	test_case( "/i pattern character class", /ba[0a-c99S-Z0]/i & /bEz/ == "bArbEz" );
+	test_case( "/i pattern character class", /ba[0a-c99M-S0]/i & /bEz/ == "bArbEz" );
+
+	test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ in "xBAry" );
+	test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ in "xFOoy" );
+	test_case( "(?i:...) pattern construct", /foo|(?i:bar)/ | /foo/i in "xFOoy" );
+
+}

testing/btest/scripts/base/frameworks/cluster/custom_pool_exclusivity.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
@@ -17,6 +16,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 global my_pool_spec: Cluster::PoolSpec =
 		Cluster::PoolSpec(
 			$topic = "bro/cluster/pool/my_pool",

testing/btest/scripts/base/frameworks/cluster/custom_pool_limits.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
@@ -17,6 +16,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 global my_pool_spec: Cluster::PoolSpec =
 		Cluster::PoolSpec(
 			$topic = "bro/cluster/pool/my_pool",

testing/btest/scripts/base/frameworks/cluster/log_distribution.bro

@@ -2,9 +2,7 @@
 #
 # @TEST-EXEC: btest-bg-run logger-1 BROPATH=$BROPATH:.. CLUSTER_NODE=logger-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run logger-2 BROPATH=$BROPATH:.. CLUSTER_NODE=logger-2 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run manager BROPATH=$BROPATH:.. CLUSTER_NODE=manager bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff logger-1/test.log
@@ -22,6 +20,10 @@ redef Cluster::nodes = {
 
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0sec;
 
 module Test;

testing/btest/scripts/base/frameworks/cluster/start-it-up-logger.bro

@@ -1,12 +1,9 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run logger-1  CLUSTER_NODE=logger-1 BROPATH=$BROPATH:.. bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run manager-1 CLUSTER_NODE=manager-1 BROPATH=$BROPATH:.. bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   CLUSTER_NODE=proxy-1 BROPATH=$BROPATH:.. bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   CLUSTER_NODE=proxy-2 BROPATH=$BROPATH:.. bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  CLUSTER_NODE=worker-1 BROPATH=$BROPATH:.. bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  CLUSTER_NODE=worker-2 BROPATH=$BROPATH:.. bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
@@ -29,6 +26,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 global fully_connected: event();
 
 global peer_count = 0;

testing/btest/scripts/base/frameworks/cluster/start-it-up.bro

@@ -1,10 +1,8 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
@@ -24,6 +22,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 global fully_connected: event();
 
 global peer_count = 0;

testing/btest/scripts/base/frameworks/cluster/topic_distribution.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
@@ -17,6 +16,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 global proxy_count = 0;
 
 event go_away()

testing/btest/scripts/base/frameworks/cluster/topic_distribution_bifs.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 30
@@ -19,6 +18,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 global proxy_count = 0;
 global q = 0;
 

testing/btest/scripts/base/frameworks/config/several-files.bro

@@ -1,7 +1,6 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait 10
-# @TEST-EXEC: sort bro/config.log >bro/config.log.tmp && mv bro/config.log.tmp bro/config.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | grep -v ^# | $SCRIPTS/diff-sort" btest-diff bro/config.log
-# @TEST-EXEC: btest-diff bro/config.log
 
 @load base/frameworks/config
 @load base/protocols/conn

testing/btest/scripts/base/frameworks/config/updates.bro

@@ -1,11 +1,11 @@
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: mv configfile2 configfile
 # @TEST-EXEC: touch configfile
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: mv configfile3 configfile
 # @TEST-EXEC: touch configfile
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: mv configfile4 configfile
 # @TEST-EXEC: touch configfile
 # @TEST-EXEC: btest-bg-wait 10
@@ -103,6 +103,12 @@ event Input::end_of_data(name: string, source:string)
 
 	eolcount += 1;
 
-	if ( eolcount == 4 )
+	if ( eolcount == 1 )
+		system("touch got1");
+	else if ( eolcount == 2 )
+		system("touch got2");
+	else if ( eolcount == 3 )
+		system("touch got3");
+	else if ( eolcount == 4 )
 		terminate();
 	}

testing/btest/scripts/base/frameworks/control/configuration_update.bro

@@ -1,13 +1,14 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Broker::default_port=65531/tcp
-# @TEST-EXEC: sleep 5
 # @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
-# @TEST-EXEC: sleep 5
-# @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff controllee/.stdout
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 const test_var = "ORIGINAL VALUE (this should be printed out first)" &redef;
 
 @TEST-START-FILE test-redef.bro
@@ -23,3 +24,8 @@ event bro_done()
 	{
 	print test_var;
 	}
+
+event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input2.log input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -75,7 +75,9 @@ event Input::end_of_data(name: string, source: string)
 	print outfile, servers;
 	
 	try = try + 1;
-	if ( try == 2 )
+	if ( try == 1 )
+		system("touch got1");
+	else if ( try == 2 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/scripts/base/frameworks/input/missing-file-initially.bro

@@ -4,12 +4,12 @@
 # failing behavior.
 
 # @TEST-EXEC: btest-bg-run bro bro %INPUT
-# @TEST-EXEC: sleep 10
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/init 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: mv does-exist.dat does-not-exist.dat
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/next 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: mv does-not-exist.dat does-not-exist-again.dat
 # @TEST-EXEC: echo "3 streaming still works" >> does-not-exist-again.dat
-# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-bg-wait 5
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stderr
 
@@ -31,9 +31,17 @@ type Val: record {
 	line: string;
 };
 
+global line_count = 0;
+
 event line(description: Input::EventDescription, tpe: Input::Event, v: Val)
 	{
 	print fmt("%s: %s", description$name, v$line);
+	++line_count;
+
+	if ( line_count == 4 )
+		system("touch next");
+	if ( line_count == 5 )
+		terminate();
 	}
 
 event line2(description: Input::EventDescription, tpe: Input::Event, v: Val)
@@ -49,4 +57,5 @@ event bro_init()
 	Input::add_event([$source="../does-not-exist.dat", $name="inputmanual", $reader=Input::READER_ASCII, $mode=Input::MANUAL, $fields=Val, $ev=line, $want_record=T]);
 	Input::add_event([$source="../does-not-exist.dat", $name="input2", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line2, $want_record=T,
 	                  $config=table(["fail_on_file_problem"] = "T")]);
+	system("touch init");
 	}

testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro

@@ -1,12 +1,12 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input2.log input.log
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input3.log input.log
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input4.log input.log
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input5.log input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -96,7 +96,15 @@ event Input::end_of_data(name: string, source: string)
 	try = try + 1;
 	print outfile, fmt("Update_finished for %s, try %d", name, try);
 	print outfile, servers;
-	
+
+	if ( try == 1 )
+		system("touch got1");
+	else if ( try == 2 )
+		system("touch got2");
+	else if ( try == 3 )
+		system("touch got3");
+	else if ( try == 4 )
+		system("touch got4");
 	if ( try == 5 )
 		{
 		close(outfile);

testing/btest/scripts/base/frameworks/input/raw/executestdin.bro

@@ -7,6 +7,7 @@ redef exit_only_after_terminate = T;
 
 global outfile: file;
 global processes_finished: count = 0;
+global lines_received: count = 0;
 global n: count = 0;
 global total_processes: count = 0;
 
@@ -20,10 +21,23 @@ type Val: record {
 	s: string;
 };
 
+global more_input: function(name_prefix: string);
+
+function check_terminate_condition()
+	{
+	if ( processes_finished != total_processes )
+		return;
+
+	if ( lines_received != (total_processes - 1) * 2 )
+		return;
+
+	terminate();
+	}
+
 event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	{
-	print outfile, tpe, description$source, description$name;
+	++lines_received;
-	print outfile, s;
+	print outfile, tpe, description$source, description$name, s;
 	}
 
 event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool)
@@ -31,10 +45,18 @@ event InputRaw::process_finished(name: string, source:string, exit_code:count, s
 	print "process_finished", name, source;
 	Input::remove(name);
 	++processes_finished;
-	if ( processes_finished == total_processes )
+	if ( processes_finished == 1 )
+		{
+		more_input("input");
+		more_input("input");
+		more_input("input");
+		more_input("input");
+		more_input("input");
+		}
+	else if ( processes_finished == total_processes )
 		{
 		close(outfile);
-		terminate();
+		check_terminate_condition();
 		}
 	}
 
@@ -59,9 +81,4 @@ event bro_init()
 	                  $reader=Input::READER_RAW, $mode=Input::STREAM,
 	                  $name="input", $fields=Val, $ev=line, $want_record=F,
 	                  $config=config_strings]);
-	more_input("input");
-	more_input("input");
-	more_input("input");
-	more_input("input");
-	more_input("input");
 	}

testing/btest/scripts/base/frameworks/input/raw/executestream.bro

@@ -1,8 +1,8 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
-# @TEST-EXEC: sleep 3
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input2.log >> input.log
-# @TEST-EXEC: sleep 3
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input3.log >> input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -38,12 +38,16 @@ global outfile: file;
 
 event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	{
-	print outfile, description;
+	print outfile, description$source, description$reader, description$mode, description$name;
 	print outfile, tpe;
 	print outfile, s;
 
 	try = try + 1;
-	if ( try == 8 )
+	if ( try == 1 )
+		system("touch got1");
+	else if ( try == 3 )
+		system("touch got3");
+	else if ( try == 8 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/scripts/base/frameworks/input/raw/offset.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: cp input.log input2.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: echo "hi" >> input2.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
@@ -24,7 +24,9 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	{
 	print outfile, s;
 	try = try + 1;
-	if ( try == 3 )
+	if ( try == 2 )
+		system("touch got2");
+	else if ( try == 3 )
 		{
 		close(outfile);
 		terminate();

testing/btest/scripts/base/frameworks/input/raw/stderr.bro

@@ -1,3 +1,4 @@
+# @TEST-EXEC: mkdir mydir && touch mydir/a && touch mydir/b && touch mydir/c
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -9,64 +10,59 @@ type Val: record {
 	is_stderr: bool;
 };
 
-global try: count;
+global try = 0;
+global n = 0;
 global outfile: file;
 
 event line(description: Input::EventDescription, tpe: Input::Event, s: string, is_stderr: bool)
 	{
-	print outfile, tpe;
+	local line_output = fmt("%s line output (stderr=%s): ", tpe, is_stderr);
+
 	if ( is_stderr ) 
 		{
 		# work around localized error messages. and if some localization does not include the filename... well... that would be bad :)
 		if ( strstr(s, "nonexistant") > 0 ) 
-			{
+			line_output += "<stderr output contained nonexistant>";
-			print outfile, "stderr output contained nonexistant";
+		else
-			}
+			line_output += "<unexpected/weird error localization>";
 		}
 	else
-		{
+		line_output += s;
-		print outfile, s;
-		}
-	print outfile, is_stderr;
 
-	try = try + 1;
+	print outfile, line_output;
-	if ( try == 7 )
+	++try;
-		{
+
-		print outfile, "done";
+	if ( n == 2 && try == 7 )
-		Input::remove("input");
+		terminate();
-		}
 	}
 
-global n = 0;
-
 event Input::end_of_data(name: string, source:string)
 	{
-	print outfile, "End of Data event";
+	print outfile, "End of Data event", name;
-	print outfile, name;
 	++n;
-	if ( n == 2 )
+
+	if ( n == 2 && try == 7 )
 		terminate();
 	}
 
 event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool)
 	{
-	print outfile, "Process finished event";
+	print outfile, "Process finished event", name, exit_code != 0;
-	print outfile, name;
-	if ( exit_code != 0 ) 
-		print outfile, "Exit code != 0";
 	++n;
-	if ( n == 2 )
+
+	if ( n == 2 && try == 7 )
 		terminate();
 	}
 
 event bro_init()
 	{
-
 	local config_strings: table[string] of string = {
 		["read_stderr"] = "1"
 	};
 
 	outfile = open("../out");
-	try = 0;
+	Input::add_event([$source="ls ../mydir ../nonexistant ../nonexistant2 ../nonexistant3 |",
-	Input::add_event([$source="ls .. ../nonexistant ../nonexistant2 ../nonexistant3 |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings, $mode=Input::STREAM]);
+	                 $reader=Input::READER_RAW, $name="input",
+	                 $fields=Val, $ev=line, $want_record=F,
+	                 $config=config_strings, $mode=Input::STREAM]);
 	}

testing/btest/scripts/base/frameworks/input/raw/streamraw.bro

@@ -1,8 +1,8 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
-# @TEST-EXEC: sleep 3
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input2.log >> input.log
-# @TEST-EXEC: sleep 3
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input3.log >> input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -37,12 +37,17 @@ global outfile: file;
 
 event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	{
-	print outfile, description;
+	print outfile, description$source, description$reader, description$mode, description$name;
 	print outfile, tpe;
 	print outfile, s;
 
 	try = try + 1;
-	if ( try == 8 )
+
+	if ( try == 1 )
+		system("touch got1");
+	else if ( try == 3 )
+		system("touch got3");
+	else if ( try == 8 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -1,12 +1,12 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input2.log input.log
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input3.log input.log
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got3 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input4.log input.log
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got4 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input5.log input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -126,7 +126,16 @@ event Input::end_of_data(name: string, source: string)
 	print outfile, servers;
 	
 	try = try + 1;
-	if ( try == 5 )
+
+	if ( try == 1 )
+		system("touch got1");
+	else if ( try == 2 )
+		system("touch got2");
+	else if ( try == 3 )
+		system("touch got3");
+	else if ( try == 4 )
+		system("touch got4");
+	else if ( try == 5 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/scripts/base/frameworks/input/stream.bro

@@ -1,8 +1,8 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 3
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input2.log >> input.log
-# @TEST-EXEC: sleep 3
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cat input3.log >> input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff out
@@ -66,8 +66,12 @@ event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, r
 	print outfile, servers;
 
 	try = try + 1;
-	
+
-	if ( try == 3 )
+	if ( try == 1 )
+		system("touch got1");
+	else if ( try == 2 )
+		system("touch got2");
+	else if ( try == 3 )
 		{
 		print outfile, "done";
 		close(outfile);

testing/btest/scripts/base/frameworks/input/twotables.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: cp input1.log input.log
 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT
-# @TEST-EXEC: sleep 5
+# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp input3.log input.log
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff event.out
@@ -116,7 +116,9 @@ event Input::end_of_data(name: string, source: string)
 	#print fin_out, servers;
 	
 	try = try + 1;
-	if ( try == 3 )
+	if ( try == 2 )
+		system("touch got2");
+	else if ( try == 3 )
 		{
 		print fin_out, "done";
 		print fin_out, servers;

testing/btest/scripts/base/frameworks/intel/input-and-match.bro

@@ -1,4 +1,3 @@
-# @TEST-SERIALIZE: comm
 
 # @TEST-EXEC: btest-bg-run broproc bro %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5

testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait -k 10
@@ -26,6 +25,10 @@ e@mail.com	Intel::EMAIL	source1	Phishing email source	http://some-data-distribut
 @TEST-END-FILE
 
 @load base/frameworks/control
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval=0sec;
 
 module Intel;

testing/btest/scripts/base/frameworks/intel/updated-match.bro

@@ -1,12 +1,10 @@
-# @TEST-SERIALIZE: comm
-
 # @TEST-EXEC: cp intel1.dat intel.dat
 # @TEST-EXEC: btest-bg-run broproc bro %INPUT
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file broproc/got1 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp intel2.dat intel.dat
-# @TEST-EXEC: sleep 2
+# @TEST-EXEC: $SCRIPTS/wait-for-file broproc/got2 5 || (btest-bg-wait -k 1 && false)
 # @TEST-EXEC: cp intel3.dat intel.dat
-# @TEST-EXEC: btest-bg-wait 6
+# @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: cat broproc/intel.log > output
 # @TEST-EXEC: cat broproc/notice.log >> output
 # @TEST-EXEC: btest-diff output
@@ -35,6 +33,8 @@ redef Intel::read_files += { "../intel.dat" };
 redef enum Intel::Where += { SOMEWHERE };
 
 global runs = 0;
+global entries_read = 0;
+
 event do_it()
 	{
 	Intel::seen([$host=1.2.3.4,
@@ -43,8 +43,11 @@ event do_it()
 	             $where=SOMEWHERE]);
 
 	++runs;
-	if ( runs < 3 )
+
-		schedule 3sec { do_it() };
+	if ( runs == 1 )
+		system("touch got1");
+	if ( runs == 2 )
+		system("touch got2");
 	}
 
 global log_lines = 0;
@@ -55,7 +58,17 @@ event Intel::log_intel(rec: Intel::Info)
 		terminate();
 	}
 
-event bro_init() &priority=-10
+module Intel;
+
+event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item)
 	{
-	schedule 1sec { do_it() };
+	++entries_read;
+	print entries_read;
+
+	if ( entries_read == 1 )
+		event do_it();
+	else if ( entries_read == 3 )
+		event do_it();
+	else if ( entries_read == 5 )
+		event do_it();
 	}

testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro

@@ -1,11 +1,10 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT"
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT"
 # @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: cat manager-1/reporter.log | grep -v "reporter/" > manager-reporter.log
-# @TEST-EXEC: btest-diff manager-reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | grep -v ^# | $SCRIPTS/diff-sort" btest-diff manager-reporter.log
 
 
 @TEST-START-FILE cluster-layout.bro
@@ -21,6 +20,10 @@ redef Cluster::nodes = {
 redef exit_only_after_terminate = T;
 @endif
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 redef Log::default_scope_sep="_";

testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT"
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT"
 # @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/http.log
@@ -20,6 +19,9 @@ redef Cluster::nodes = {
 redef exit_only_after_terminate = T;
 @endif
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
 redef Log::default_rotation_interval = 0secs;
 
 redef Log::default_scope_sep="_";
@@ -59,9 +61,12 @@ event bro_init()
 	{
 	if ( Cluster::node == "worker-1" )
 		Broker::subscribe("death");
+	}
 
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+	{
 	if ( Cluster::node == "manager-1" )
-		schedule 13sec { kill_worker() };
+		schedule 2sec { kill_worker() };
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)

testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro

@@ -11,22 +11,35 @@
 @load base/frameworks/netcontrol
 
 redef exit_only_after_terminate = T;
+global have_peer = F;
+global did_init = F;
+
+event bro_init()
+	{
+	suspend_processing();
+	}
 
 event NetControl::init()
 	{
-	suspend_processing();
 	local netcontrol_acld = NetControl::create_acld(NetControl::AcldConfig($acld_host=127.0.0.1, $acld_port=Broker::default_port, $acld_topic="bro/event/netcontroltest"));
 	NetControl::activate(netcontrol_acld, 0);
 	}
 
 event NetControl::init_done()
 	{
-	continue_processing();
+	did_init = T;
+
+	if ( did_init && have_peer )
+		continue_processing();
 	}
 
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Broker peer added", endpoint$network;
+	have_peer = T;
+
+	if ( did_init && have_peer )
+		continue_processing();
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)

testing/btest/scripts/base/frameworks/netcontrol/acld.bro

@@ -12,10 +12,16 @@
 @load base/frameworks/netcontrol
 
 redef exit_only_after_terminate = T;
+global have_peer = F;
+global did_init = F;
+
+event bro_init()
+	{
+	suspend_processing();
+	}
 
 event NetControl::init()
 	{
-	suspend_processing();
 	local netcontrol_acld = NetControl::create_acld(NetControl::AcldConfig($acld_host=127.0.0.1, $acld_port=Broker::default_port, $acld_topic="bro/event/netcontroltest"));
 	NetControl::activate(netcontrol_acld, 0);
 	}
@@ -23,11 +29,18 @@ event NetControl::init()
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Broker peer added", endpoint$network;
+	have_peer = T;
+
+	if ( did_init && have_peer )
+		continue_processing();
 	}
 
 event NetControl::init_done()
 	{
-	continue_processing();
+	did_init = T;
+
+	if ( did_init && have_peer )
+		continue_processing();
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)

testing/btest/scripts/base/frameworks/netcontrol/basic-cluster.bro

@@ -1,9 +1,10 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT"
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT"
-# @TEST-EXEC: sleep 1
+
+# @TEST-EXEC: $SCRIPTS/wait-for-pid $(cat worker-1/.pid) 10 || (btest-bg-wait -k 1 && false)
+
 # @TEST-EXEC: btest-bg-run worker-2  "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT"
 # @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff worker-1/.stdout
@@ -17,6 +18,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 #redef exit_only_after_terminate = T;
 
@@ -51,9 +56,14 @@ event terminate_me() {
 	terminate();
 }
 
+global peers_lost = 0;
+
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
-	schedule 1sec { terminate_me() };
+	++peers_lost;
+
+	if ( peers_lost == 2 )
+		schedule 2sec { terminate_me() };
 	}
 
 event NetControl::rule_added(r: NetControl::Rule, p: NetControl::PluginState, msg: string &default="")

testing/btest/scripts/base/frameworks/netcontrol/broker.bro

@@ -12,22 +12,35 @@
 @load base/frameworks/netcontrol
 
 redef exit_only_after_terminate = T;
+global have_peer = F;
+global did_init = F;
+
+event bro_init()
+	{
+	suspend_processing();
+	}
 
 event NetControl::init()
 	{
-	suspend_processing();
 	local netcontrol_broker = NetControl::create_broker(NetControl::BrokerConfig($host=127.0.0.1, $bport=Broker::default_port, $topic="bro/event/netcontroltest"), T);
 	NetControl::activate(netcontrol_broker, 0);
 	}
 
 event NetControl::init_done()
 	{
-	continue_processing();
+	did_init = T;
+
+	if ( did_init && have_peer )
+		continue_processing();
 	}
 
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Broker peer added", endpoint$network;
+	have_peer = T;
+
+	if ( did_init && have_peer )
+		continue_processing();
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)

testing/btest/scripts/base/frameworks/notice/cluster.bro

@@ -2,7 +2,6 @@
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
@@ -15,6 +14,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 redef enum Notice::Type += {

testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro

@@ -2,7 +2,6 @@
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 20
@@ -17,6 +16,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 redef enum Notice::Type += {

testing/btest/scripts/base/frameworks/openflow/log-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT"
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/smtp.trace %INPUT"
 # @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/openflow.log
@@ -13,6 +12,9 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
 redef Log::default_rotation_interval = 0secs;
 #redef exit_only_after_terminate = T;
 
@@ -21,6 +23,18 @@ redef Log::default_rotation_interval = 0secs;
 
 global of_controller: OpenFlow::Controller;
 
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+event bro_init()
+	{
+	suspend_processing();
+	}
+
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	continue_processing();
+	}
+@endif
+
 event bro_init()
 	{
 	of_controller = OpenFlow::log_new(42);

testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 15
@@ -16,6 +15,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 global n = 0;

testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 3
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 20
@@ -15,6 +14,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 event bro_init() &priority=5

testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 15
@@ -17,6 +16,10 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
+
 redef Log::default_rotation_interval = 0secs;
 
 global n = 0;

testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 15
@@ -15,6 +14,9 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
 redef Log::default_rotation_interval = 0secs;
 
 event bro_init() &priority=5

testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro

@@ -1,7 +1,6 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 15
@@ -16,6 +15,9 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+redef Cluster::retry_interval = 1sec;
+redef Broker::default_listen_retry = 1sec;
+redef Broker::default_connect_retry = 1sec;
 redef Log::default_rotation_interval = 0secs;
 
 

testing/scripts/diff-remove-timestamps

@@ -9,6 +9,5 @@ else
    sed="sed -E"
 fi
 
-# The first sed uses a "basic" regexp, the 2nd a "modern:.
+$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
-sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \
 $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

testing/scripts/travis-job

@@ -143,7 +143,7 @@ run() {
         chmod 600 travis_key
         mkdir -p ~/.ssh
         mv travis_key ~/.ssh/id_rsa
-        ssh-keyscan -H -p 22 -t rsa git.bro.org >> ~/.ssh/known_hosts
+        echo "git.bro.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmlu+EaJfPKTVqoEUzb5JBEdvNiFxO2wm7Vl61dGBl57avakFl8YnRujbA2yxlpC2xnEKD5y++hXxtxRLefyCM=" >> ~/.ssh/known_hosts
         git clone ssh://git@git.bro.org/bro-testing-private
         rm ~/.ssh/id_rsa
     elif [ -n "${TRAVIS_PULL_REQUEST}" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then

testing/scripts/wait-for-file

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+# Sleeps until a file comes into existence.
+
+if [[ $# -ne 2 ]]; then
+  >&2 echo "usage: $0 <file to wait for> <max secs to wait>"
+  exit 1
+fi
+
+wait_file=$1
+max_wait=$2
+wait_count=0
+
+while [[ ! -e $wait_file ]]; do
+  let "wait_count += 1"
+
+  if [[ $wait_count -ge $max_wait ]]; then
+    >&2 echo "error: file '$wait_file' does not exist after $max_wait seconds"
+    exit 1
+  fi
+
+  sleep 1
+done
+

testing/scripts/wait-for-pid

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+# Sleeps until a process id no longer exists.
+
+if [[ $# -ne 2 ]]; then
+  >&2 echo "usage: $0 <pid to wait for> <max secs to wait>"
+  exit 1
+fi
+
+wait_pid=$1
+max_wait=$2
+wait_count=0
+
+while kill -0 $wait_pid &> /dev/null; do
+  let "wait_count += 1"
+
+  if [[ $wait_count -ge $max_wait ]]; then
+    >&2 echo "error: process $wait_pid still exists after $max_wait seconds"
+    exit 1
+  fi
+
+  sleep 1
+done
+