Merge remote-tracking branch 'origin/master' into topic/seth/intel-framework

CHANGES

@@ -1,4 +1,47 @@
 
+2.1-58 | 2012-10-08 10:10:09 -0700
+
+  * Fix a problem with non-manager cluster nodes applying
+    Notice::policy. This could, for example, result in duplicate
+    emails being sent if Notice::emailed_types is redef'd in local.bro
+    (or any script that gets loaded on all cluster nodes). (Jon Siwek)
+
+2.1-56 | 2012-10-03 16:04:52 -0700
+
+  * Add general FAQ entry about upgrading Bro. (Jon Siwek)
+
+2.1-53 | 2012-10-03 16:00:40 -0700
+
+  * Add new Tunnel::delay_teredo_confirmation option that indicates
+    that the Teredo analyzer should wait until it sees both sides of a
+    connection using a valid Teredo encapsulation before issuing a
+    protocol_confirmation. Default is on. Addresses #890. (Jon Siwek)
+
+2.1-50 | 2012-10-02 12:06:08 -0700
+
+  * Fix a typing issue that prevented the ElasticSearch timeout to
+    work. (Matthias Vallentin)
+
+  * Use second granularity for ElasticSearch timeouts. (Matthias
+    Vallentin)
+
+  * Fix compile issues with older versions of libcurl, which don't
+    offer *_MS timeout constants. (Matthias Vallentin)
+
+2.1-47 | 2012-10-02 11:59:29 -0700
+
+  * Fix for the input framework: BroStrings were constructed without a
+    final \0, which makes them unusable by basically all internal
+    functions (like to_count). (Bernhard Amann)
+
+  * Remove deprecated script functionality (see NEWS for details).
+    (Daniel Thayer)
+    
+2.1-39 | 2012-09-29 14:09:16 -0700
+
+  * Reliability adjustments to istate tests with network
+    communication. (Jon Siwek)
+
 2.1-37 | 2012-09-25 14:21:37 -0700
 
   * Reenable some tests that previously would cause Bro to exit with

NEWS

@@ -18,8 +18,19 @@ New Functionality
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~
 
-- TODO: Update.
+- We removed the following, already deprecated, functionality:
 
+    * Scripting language:
+        - &disable_print_hook attribute.
+
+    * BiF functions:
+        - parse_dotted_addr(), dump_config(),
+          make_connection_persistent(), generate_idmef(),
+          split_complete()
+
+- Removed a now unused argument from "do_split" helper function.
+
+- "this" is no longer a reserved keyword.
 
 Bro 2.1
 -------

VERSION

@@ -1 +1 @@
-2.1-37
+2.1-58

aux/broctl

@@ -1 +1 @@
-Subproject commit 44afce440d02e1aac4012d5b0f5a26875ae11c3e
+Subproject commit b0e3c0d84643878c135dcb8a9774ed78147dd648

cmake

@@ -1 +1 @@
-Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130
+Subproject commit 125f9a5fa851381d0350efa41a4d14f27be263a2

doc/ext/bro_lexer/bro.py

@@ -29,7 +29,7 @@ class BroLexer(RegexLexer):
              r'|vector)\b', Keyword.Type),
             (r'(T|F)\b', Keyword.Constant),
             (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
-             r'|default|disable_print_hook|raw_output|encrypt|group|log'
+             r'|default|raw_output|encrypt|group|log'
              r'|mergeable|optional|persistent|priority|redef'
              r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
                  Keyword)),

doc/faq.rst

@@ -12,6 +12,43 @@ Frequently Asked Questions
 Installation and Configuration
 ==============================
 
+How do I upgrade to a new version of Bro?
+-----------------------------------------
+
+There's two suggested approaches, either install Bro using the same
+installation prefix directory as before, or pick a new prefix and copy
+local customizations over.
+
+Re-Use Previous Install Prefix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you choose to configure and install Bro with the same prefix
+directory as before, local customization and configuration to files in
+``$prefix/share/bro/site`` and ``$prefix/etc`` won't be overwritten
+(``$prefix`` indicating the root of where Bro was installed). Also, logs
+generated at run-time won't be touched by the upgrade. (But making
+a backup of local changes before proceeding is still recommended.)
+
+After upgrading, remember to check ``$prefix/share/bro/site`` and
+``$prefix/etc`` for ``.example`` files, which indicate the
+distribution's version of the file differs from the local one, which may
+include local changes.  Review the differences, and make adjustments
+as necessary (for differences that aren't the result of a local change,
+use the new version's).
+
+Pick a New Install prefix
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you want to install the newer version in a different prefix
+directory than before, you can just copy local customization and
+configuration files from ``$prefix/share/bro/site`` and ``$prefix/etc``
+to the new location (``$prefix`` indicating the root of where Bro was
+originally installed).  Make sure to review the files for difference
+before copying and make adjustments as necessary (for differences that
+aren't the result of a local change, use the new version's).  Of
+particular note, the copied version of ``$prefix/etc/broctl.cfg`` is
+likely to need changes to the ``SpoolDir`` and ``LogDir`` settings.
+
 How can I tune my operating system for best capture performance?
 ----------------------------------------------------------------
 

doc/scripts/builtins.rst

@@ -600,10 +600,6 @@ scripting language supports the following built-in attributes.
 
 .. TODO: needs to be documented.
 
-.. bro:attr:: &disable_print_hook
-
-    Deprecated. Will be removed.
-
 .. bro:attr:: &raw_output
 
     Opens a file in raw mode, i.e., non-ASCII characters are not

scripts/base/frameworks/logging/writers/elasticsearch.bro

@@ -26,8 +26,10 @@ export {
 	## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
 	const type_prefix = "" &redef;
 
-	## The time before an ElasticSearch transfer will timeout.
+	## The time before an ElasticSearch transfer will timeout. Note that
-	## This is not working!
+	## the fractional part of the timeout will be ignored. In particular, time
+	## specifications less than a second result in a timeout value of 0, which
+	## means "no timeout."
 	const transfer_timeout = 2secs;
 
 	## The batch size is the number of messages that will be queued up before

scripts/base/frameworks/notice/cluster.bro

@@ -23,7 +23,7 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # The notice policy is completely handled by the manager and shouldn't be 
 # done by workers or proxies to save time for packet processing.
-event bro_init() &priority=-11
+event bro_init() &priority=11
 	{
 	Notice::policy = table();
 	}

scripts/base/init-bare.bro

@@ -2784,6 +2784,14 @@ export {
 	## to have a valid Teredo encapsulation.
 	const yielding_teredo_decapsulation = T &redef;
 
+	## With this set, the Teredo analyzer waits until it sees both sides
+	## of a connection using a valid Teredo encapsulation before issuing
+	## a :bro:see:`protocol_confirmation`.  If it's false, the first
+	## occurence of a packet with valid Teredo encapsulation causes a
+	## confirmation.  Both cases are still subject to effects of
+	## :bro:see:`Tunnel::yielding_teredo_decapsulation`.
+	const delay_teredo_confirmation = T &redef;
+
 	## How often to cleanup internal state for inactive IP tunnels.
 	const ip_tunnel_timeout = 24hrs &redef;
 } # end export

src/Attr.cc

@@ -15,7 +15,7 @@ const char* attr_name(attr_tag t)
 		"&add_func", "&delete_func", "&expire_func",
 		"&read_expire", "&write_expire", "&create_expire",
 		"&persistent", "&synchronized", "&postprocessor",
-		"&encrypt", "&match", "&disable_print_hook",
+		"&encrypt", "&match",
 		"&raw_output", "&mergeable", "&priority",
 		"&group", "&log", "&error_handler", "&type_column",
 		"(&tracked)",
@@ -385,11 +385,6 @@ void Attributes::CheckAttr(Attr* a)
 		// FIXME: Check here for global ID?
 		break;
 
-	case ATTR_DISABLE_PRINT_HOOK:
-		if ( type->Tag() != TYPE_FILE )
-			Error("&disable_print_hook only applicable to files");
-		break;
-
 	case ATTR_RAW_OUTPUT:
 		if ( type->Tag() != TYPE_FILE )
 			Error("&raw_output only applicable to files");

src/Attr.h

@@ -28,7 +28,6 @@ typedef enum {
 	ATTR_POSTPROCESSOR,
 	ATTR_ENCRYPT,
 	ATTR_MATCH,
-	ATTR_DISABLE_PRINT_HOOK,
 	ATTR_RAW_OUTPUT,
 	ATTR_MERGEABLE,
 	ATTR_PRIORITY,

src/File.cc

@@ -514,9 +514,6 @@ void BroFile::SetAttrs(Attributes* arg_attrs)
 			InitEncrypt(log_encryption_key->AsString()->CheckString());
 		}
 
-	if ( attrs->FindAttr(ATTR_DISABLE_PRINT_HOOK) )
-		DisablePrintHook();
-
 	if ( attrs->FindAttr(ATTR_RAW_OUTPUT) )
 		EnableRawOutput();
 

src/File.h

@@ -57,7 +57,7 @@ public:
 	RecordVal* Rotate();
 
 	// Set &rotate_interval, &rotate_size, &postprocessor,
-	// &disable_print_hook, and &raw_output attributes.
+	// and &raw_output attributes.
 	void SetAttrs(Attributes* attrs);
 
 	// Returns the current size of the file, after fresh stat'ing.

src/Teredo.cc

@@ -138,6 +138,11 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 	{
 	Analyzer::DeliverPacket(len, data, orig, seq, ip, caplen);
 
+	if ( orig )
+		valid_orig = false;
+	else
+		valid_resp = false;
+
 	TeredoEncapsulation te(this);
 
 	if ( ! te.Parse(data, len) )
@@ -150,7 +155,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 
 	if ( e && e->Depth() >= BifConst::Tunnel::max_depth )
 		{
-		Weird("tunnel_depth");
+		Weird("tunnel_depth", true);
 		return;
 		}
 
@@ -162,7 +167,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 		if ( inner->NextProto() == IPPROTO_NONE && inner->PayloadLen() == 0 )
 			// Teredo bubbles having data after IPv6 header isn't strictly a
 			// violation, but a little weird.
-			Weird("Teredo_bubble_with_payload");
+			Weird("Teredo_bubble_with_payload", true);
 		else
 			{
 			delete inner;
@@ -173,6 +178,11 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 
 	if ( rslt == 0 || rslt > 0 )
 		{
+		if ( orig )
+			valid_orig = true;
+		else
+			valid_resp = true;
+
 		if ( BifConst::Tunnel::yielding_teredo_decapsulation &&
 		     ! ProtocolConfirmed() )
 			{
@@ -193,7 +203,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 				}
 
 			if ( ! sibling_has_confirmed )
-				ProtocolConfirmation();
+				Confirm();
 			else
 				{
 				delete inner;
@@ -201,10 +211,8 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 				}
 			}
 		else
-			{
+			// Aggressively decapsulate anything with valid Teredo encapsulation.
-			// Aggressively decapsulate anything with valid Teredo encapsulation
+			Confirm();
-			ProtocolConfirmation();
-			}
 		}
 
 	else

src/Teredo.h

@@ -6,7 +6,8 @@
 
 class Teredo_Analyzer : public Analyzer {
 public:
-	Teredo_Analyzer(Connection* conn) : Analyzer(AnalyzerTag::Teredo, conn)
+	Teredo_Analyzer(Connection* conn) : Analyzer(AnalyzerTag::Teredo, conn),
+	                                    valid_orig(false), valid_resp(false)
 		{}
 
 	virtual ~Teredo_Analyzer()
@@ -26,18 +27,34 @@ public:
 
 	/**
 	 * Emits a weird only if the analyzer has previously been able to
-	 * decapsulate a Teredo packet since otherwise the weirds could happen
+	 * decapsulate a Teredo packet in both directions or if *force* param is
-	 * frequently enough to be less than helpful.
+	 * set, since otherwise the weirds could happen frequently enough to be less
+	 * than helpful.  The *force* param is meant for cases where just one side
+	 * has a valid encapsulation and so the weird would be informative.
 	 */
-	void Weird(const char* name) const
+	void Weird(const char* name, bool force = false) const
 		{
-		if ( ProtocolConfirmed() )
+		if ( ProtocolConfirmed() || force )
 			reporter->Weird(Conn(), name);
 		}
 
+	/**
+	 * If the delayed confirmation option is set, then a valid encapsulation
+	 * seen from both end points is required before confirming.
+	 */
+	void Confirm()
+		{
+		if ( ! BifConst::Tunnel::delay_teredo_confirmation ||
+		     ( valid_orig && valid_resp ) )
+			ProtocolConfirmation();
+		}
+
 protected:
 	friend class AnalyzerTimer;
 	void ExpireTimer(double t);
+
+	bool valid_orig;
+	bool valid_resp;
 };
 
 class TeredoEncapsulation {

src/bro.bif

@@ -4858,7 +4858,7 @@ function file_size%(f: string%) : double
 	%}
 
 ## Disables sending :bro:id:`print_hook` events to remote peers for a given
-## file. This function is equivalent to :bro:attr:`&disable_print_hook`. In a
+## file. In a
 ## distributed setup, communicating Bro instances generate the event
 ## :bro:id:`print_hook` for each print statement and send it to the remote
 ## side. When disabled for a particular file, these events will not be
@@ -4874,7 +4874,7 @@ function disable_print_hook%(f: file%): any
 	%}
 
 ## Prevents escaping of non-ASCII characters when writing to a file.
-## This function is equivalent to :bro:attr:`&disable_print_hook`.
+## This function is equivalent to :bro:attr:`&raw_output`.
 ##
 ## f: The file to disable raw output for.
 ##
@@ -5683,12 +5683,6 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
 #
 # ===========================================================================
 
-## Deprecated. Will be removed.
-function parse_dotted_addr%(s: string%): addr
-	%{
-	IPAddr a(s->CheckString());
-	return new AddrVal(a);
-	%}
 
 
 %%{
@@ -5788,75 +5782,3 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
 		}
 	%}
 
-## Deprecated. Will be removed.
-function dump_config%(%) : bool
-	%{
-	return new Val(persistence_serializer->WriteConfig(true), TYPE_BOOL);
-	%}
-
-## Deprecated. Will be removed.
-function make_connection_persistent%(c: connection%) : any
-	%{
-	c->MakePersistent();
-	return 0;
-	%}
-
-%%{
-// Experimental code to add support for IDMEF XML output based on
-// notices.  For now, we're implementing it as a builtin you can call on an
-// notices record.
-
-#ifdef USE_IDMEF
-extern "C" {
-#include <libidmef/idmefxml.h>
-}
-#endif
-
-#include <sys/socket.h>
-
-char* port_to_string(PortVal* port)
-	{
-	char buf[256];	// to hold sprintf results on port numbers
-	snprintf(buf, sizeof(buf), "%u", port->Port());
-	return copy_string(buf);
-	}
-
-%%}
-
-## Deprecated. Will be removed.
-function generate_idmef%(src_ip: addr, src_port: port,
-				dst_ip: addr, dst_port: port%) : bool
-	%{
-#ifdef USE_IDMEF
-	xmlNodePtr message =
-		newIDMEF_Message(newAttribute("version","1.0"),
-			newAlert(newCreateTime(NULL),
-			newSource(
-				newNode(newAddress(
-					newAttribute("category","ipv4-addr"),
-					newSimpleElement("address",
-					copy_string(src_ip->AsAddr().AsString().c_str())),
-					NULL), NULL),
-				newService(
-					newSimpleElement("port",
-						port_to_string(src_port)),
-					NULL), NULL),
-			newTarget(
-				newNode(newAddress(
-					newAttribute("category","ipv4-addr"),
-					newSimpleElement("address",
-					copy_string(dst_ip->AsAddr().AsString().c_str())),
-					NULL), NULL),
-				newService(
-					newSimpleElement("port",
-						port_to_string(dst_port)),
-					NULL), NULL), NULL), NULL);
-
-	// if ( validateCurrentDoc() )
-	printCurrentMessage(stderr);
-	return new Val(1, TYPE_BOOL);
-#else
-	builtin_error("Bro was not configured for IDMEF support");
-	return new Val(0, TYPE_BOOL);
-#endif
-	%}

src/const.bif

@@ -16,6 +16,7 @@ const Tunnel::enable_ip: bool;
 const Tunnel::enable_ayiya: bool;
 const Tunnel::enable_teredo: bool;
 const Tunnel::yielding_teredo_decapsulation: bool;
+const Tunnel::delay_teredo_confirmation: bool;
 const Tunnel::ip_tunnel_timeout: interval;
 
 const Threading::heartbeat_interval: interval;

src/input/Manager.cc

@@ -2007,7 +2007,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 
 	case TYPE_STRING:
 		{
-		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 0);
+		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 1);
 		return new StringVal(s);
 		}
 

src/logging/writers/ElasticSearch.cc

@@ -48,7 +48,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	last_send = current_time();
 	failing = false;
 
-	transfer_timeout = BifConst::LogElasticSearch::transfer_timeout * 1000;
+	transfer_timeout = static_cast<long>(BifConst::LogElasticSearch::transfer_timeout);
 
 	curl_handle = HTTPSetup();
 }
@@ -373,8 +373,8 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 
 	// Some timeout options.  These will need more attention later.
 	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
-	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout);
+	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout);
-	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2);
+	curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout);
 	curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60);
 
 	CURLcode return_code = curl_easy_perform(handle);

src/logging/writers/ElasticSearch.h

@@ -68,7 +68,7 @@ private:
 
 	string path;
 	string index_prefix;
-	uint64 transfer_timeout;
+	long transfer_timeout;
 	bool failing;
 
 	uint64 batch_size;

src/parse.y

@@ -2,7 +2,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 90
+%expect 87
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -14,7 +14,7 @@
 %token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT
 %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF
 %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET
-%token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE TOK_THIS
+%token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE
 %token TOK_TIME TOK_TIMEOUT TOK_TIMER TOK_TYPE TOK_UNION TOK_VECTOR TOK_WHEN
 
 %token TOK_ATTR_ADD_FUNC TOK_ATTR_ATTR TOK_ATTR_ENCRYPT TOK_ATTR_DEFAULT
@@ -22,7 +22,7 @@
 %token TOK_ATTR_ROTATE_SIZE TOK_ATTR_DEL_FUNC TOK_ATTR_EXPIRE_FUNC
 %token TOK_ATTR_EXPIRE_CREATE TOK_ATTR_EXPIRE_READ TOK_ATTR_EXPIRE_WRITE
 %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED
-%token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE
+%token TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE
 %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER
 %token TOK_ATTR_TYPE_COLUMN
 
@@ -118,7 +118,6 @@ extern const char* g_curr_debug_error;
 
 #define YYLTYPE yyltype
 
-Expr* bro_this = 0;
 int in_init = 0;
 int in_record = 0;
 bool resolving_global_ID = false;
@@ -584,12 +583,6 @@ expr:
 			$$ = new ConstExpr(new PatternVal($1));
 			}
 
-	|	TOK_THIS
-			{
-			set_location(@1);
-			$$ = bro_this->Ref();
-			}
-
 	|       '|' expr '|'
 			{
 			set_location(@1, @3);
@@ -1297,8 +1290,6 @@ attr:
 			{ $$ = new Attr(ATTR_ENCRYPT); }
 	|	TOK_ATTR_ENCRYPT '=' expr
 			{ $$ = new Attr(ATTR_ENCRYPT, $3); }
-	|	TOK_ATTR_DISABLE_PRINT_HOOK
-			{ $$ = new Attr(ATTR_DISABLE_PRINT_HOOK); }
 	|	TOK_ATTR_RAW_OUTPUT
 			{ $$ = new Attr(ATTR_RAW_OUTPUT); }
 	|	TOK_ATTR_MERGEABLE

src/scan.l

@@ -306,7 +306,6 @@ string	return TOK_STRING;
 subnet	return TOK_SUBNET;
 switch	return TOK_SWITCH;
 table	return TOK_TABLE;
-this	return TOK_THIS;
 time	return TOK_TIME;
 timeout	return TOK_TIMEOUT;
 timer	return TOK_TIMER;
@@ -320,7 +319,6 @@ when	return TOK_WHEN;
 &create_expire	return TOK_ATTR_EXPIRE_CREATE;
 &default	return TOK_ATTR_DEFAULT;
 &delete_func	return TOK_ATTR_DEL_FUNC;
-&disable_print_hook	return TOK_ATTR_DISABLE_PRINT_HOOK;
 &raw_output return TOK_ATTR_RAW_OUTPUT;
 &encrypt	return TOK_ATTR_ENCRYPT;
 &error_handler	return TOK_ATTR_ERROR_HANDLER;

src/strings.bif

@@ -311,15 +311,9 @@ static int match_prefix(int s_len, const char* s, int t_len, const char* t)
 	return 1;
 	}
 
-Val* do_split(StringVal* str_val, RE_Matcher* re, TableVal* other_sep,
+Val* do_split(StringVal* str_val, RE_Matcher* re, int incl_sep, int max_num_sep)
-		int incl_sep, int max_num_sep)
 	{
 	TableVal* a = new TableVal(string_array);
-	ListVal* other_strings = 0;
-
-	if ( other_sep && other_sep->Size() > 0 )
-		other_strings = other_sep->ConvertToPureList();
-
 	const u_char* s = str_val->Bytes();
 	int n = str_val->Len();
 	const u_char* end_of_s = s + n;
@@ -373,9 +367,6 @@ Val* do_split(StringVal* str_val, RE_Matcher* re, TableVal* other_sep,
 			reporter->InternalError("RegMatch in split goes beyond the string");
 		}
 
-	if ( other_strings )
-		delete other_strings;
-
 	return a;
 	}
 
@@ -483,7 +474,7 @@ Val* do_sub(StringVal* str_val, RE_Matcher* re, StringVal* repl, int do_all)
 ##
 function split%(str: string, re: pattern%): string_array
 	%{
-	return do_split(str, re, 0, 0, 0);
+	return do_split(str, re, 0, 0);
 	%}
 
 ## Splits a string *once* into a two-element array of strings according to a
@@ -503,7 +494,7 @@ function split%(str: string, re: pattern%): string_array
 ## .. bro:see:: split split_all split_n str_split
 function split1%(str: string, re: pattern%): string_array
 	%{
-	return do_split(str, re, 0, 0, 1);
+	return do_split(str, re, 0, 1);
 	%}
 
 ## Splits a string into an array of strings according to a pattern. This
@@ -523,7 +514,7 @@ function split1%(str: string, re: pattern%): string_array
 ## .. bro:see:: split split1 split_n str_split
 function split_all%(str: string, re: pattern%): string_array
 	%{
-	return do_split(str, re, 0, 1, 0);
+	return do_split(str, re, 1, 0);
 	%}
 
 ## Splits a string a given number of times into an array of strings according
@@ -549,16 +540,7 @@ function split_all%(str: string, re: pattern%): string_array
 function split_n%(str: string, re: pattern,
 		incl_sep: bool, max_num_sep: count%): string_array
 	%{
-	return do_split(str, re, 0, incl_sep, max_num_sep);
+	return do_split(str, re, incl_sep, max_num_sep);
-	%}
-
-## Deprecated. Will be removed.
-# Reason: the parameter ``other`` does nothing.
-function split_complete%(str: string,
-		re: pattern, other: string_set,
-		incl_sep: bool, max_num_sep: count%): string_array
-	%{
-	return do_split(str, re, other->AsTableVal(), incl_sep, max_num_sep);
 	%}
 
 ## Substitutes a given replacement string for the first occurrence of a pattern

testing/btest/Baseline/core.tunnels.false-teredo/dpd.log

@@ -1,15 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	dpd
-#open	2009-11-18-17-59-51
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	analyzer	failure_reason
-#types	time	string	addr	port	addr	port	enum	string	string
-1258567191.486869	UWkUyAuUGXf	192.168.1.105	57696	192.168.1.1	53	udp	TEREDO	Teredo payload length [c\x1d\x81\x80\x00\x01\x00\x02\x00\x02\x00\x00\x04amch\x0equestionmarket\x03com\x00\x00\x01\x00...]
-1258578181.516140	nQcgTWjvg4c	192.168.1.104	64838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xfd\x81\x80\x00\x01\x00\x02\x00\x03\x00\x02\x08football\x02uk\x07reuters\x03com\x00\x00\x01\x00...]
-1258579063.784919	j4u32Pc5bif	192.168.1.104	55778	192.168.1.1	53	udp	TEREDO	Teredo payload length [j\x12\x81\x80\x00\x01\x00\x02\x00\x04\x00\x00\x08fastflip\x0agooglelabs\x03com\x00\x00\x01\x00...]
-1258581768.898165	TEfuqmmG4bh	192.168.1.104	50798	192.168.1.1	53	udp	TEREDO	Teredo payload length [o\xe3\x81\x80\x00\x01\x00\x02\x00\x04\x00\x04\x03www\x0fnashuatelegraph\x03com\x00\x00\x01\x00...]
-1258584478.989528	FrJExwHcSal	192.168.1.104	64963	192.168.1.1	53	udp	TEREDO	Teredo payload length [e\xbd\x81\x80\x00\x01\x00\x08\x00\x06\x00\x06\x08wellness\x05blogs\x04time\x03com\x00\x00\x01\x00...]
-1258600683.934672	5OKnoww6xl4	192.168.1.103	59838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xf0\x81\x80\x00\x01\x00\x01\x00\x02\x00\x00\x06update\x0csanasecurity\x03com\x00\x00\x01\x00...]
-#close	2009-11-19-03-18-03

testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_services
+#open	2012-10-02-20-10-05
+#fields	ts	host	port_num	port_proto	service
+#types	time	addr	port	enum	table[string]
+1258567191.405770	192.168.1.1	53	udp	TEREDO
+#close	2012-10-02-20-10-05

testing/btest/Baseline/core.tunnels.teredo/conn.log

@@ -22,7 +22,7 @@
 1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	udp	teredo	8.928880	129	48	SF	-	0	Dd	2	185	1	76	(empty)
 1210953060.829233	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	udp	teredo	13.293994	2359	11243	SF	-	0	Dd	12	2695	13	11607	(empty)
 1210953058.933954	iE6yhOq3SF	0.0.0.0	68	255.255.255.255	67	udp	-	-	-	-	S0	-	0	D	1	328	0	0	(empty)
-1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	udp	teredo	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
+1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	udp	-	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
 1210953046.591933	UWkUyAuUGXf	192.168.2.16	138	192.168.2.255	138	udp	-	28.448321	416	0	S0	-	0	D	2	472	0	0	(empty)
 1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
 1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log

@@ -9,7 +9,7 @@
 1340127577.354166	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	tcp	http	0.052829	1675	10467	S1	-	0	ShADad	10	2279	12	11191	j4u32Pc5bif
 1340127577.336558	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	udp	teredo	0.010291	129	52	SF	-	0	Dd	2	185	1	80	(empty)
 1340127577.341510	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	udp	teredo	0.065485	2367	11243	SF	-	0	Dd	12	2703	13	11607	(empty)
-1340127577.339015	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	udp	teredo	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
+1340127577.339015	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	udp	-	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
 1340127577.339015	nQcgTWjvg4c	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	k6kgXLOoSKl
 1340127577.343969	TEfuqmmG4bh	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.007778	4	4	OTH	-	0	-	1	52	1	52	UWkUyAuUGXf,j4u32Pc5bif
 1340127577.336558	arKYeMETxOg	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	UWkUyAuUGXf

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2012-06-19-17-39-37
+#open	2012-10-02-16-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
+1340127577.341510	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
 1340127577.346849	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Teredo_bubble_with_payload	-	F	bro
-1340127577.349292	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
+#close	2012-10-02-16-53-03
-#close	2012-06-19-17-39-37

testing/btest/Baseline/scripts.base.frameworks.input.basic/out

@@ -1,5 +1,5 @@
 {
-[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={
 2,
 4,
 1,
@@ -12,3 +12,4 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 }
+4242

testing/btest/core/tunnels/false-teredo.bro

@@ -1,8 +1,23 @@
 # @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap %INPUT >output
 # @TEST-EXEC: test ! -e weird.log
+# @TEST-EXEC: test ! -e dpd.log
 # @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap %INPUT Tunnel::yielding_teredo_decapsulation=F >output
 # @TEST-EXEC: btest-diff weird.log
-# @TEST-EXEC: btest-diff dpd.log
+# @TEST-EXEC: test ! -e dpd.log
+
+# In the first case, there isn't any weird or protocol violation logged
+# since the teredo analyzer recognizes that the DNS analyzer has confirmed
+# the protocol and yields.
+
+# In the second case, there are weirds since the teredo analyzer decapsulates
+# despite the presence of the confirmed DNS analyzer and the resulting
+# inner packets are malformed (no surprise there).  There's also no dpd.log
+# since the teredo analyzer doesn't confirm until it's seen a valid teredo
+# encapsulation in both directions and protocol violations aren't logged
+# until there's been a confirmation.
+
+# In either case, the analyzer doesn't, by default, get disabled as a result
+# of the protocol violations.
 
 function print_teredo(name: string, outer: connection, inner: teredo_hdr)
 	{

testing/btest/core/tunnels/teredo-known-services.test

@@ -0,0 +1,11 @@
+# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: test ! -e known_services.log
+# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: btest-diff known_services.log
+
+# The first case using Tunnel::delay_teredo_confirmation=T doesn't produce
+# a known services.log since valid Teredo encapsulations from both endpoints
+# of a connection is never witnessed and a protocol_confirmation never issued.
+
+# The second case issues protocol_confirmations more hastily and so bogus
+# entries in known-services.log are more likely to appear.

testing/btest/istate/bro-ipv6-socket.bro

@@ -4,7 +4,7 @@
 #
 # @TEST-EXEC: btest-bg-run recv bro -b ../recv.bro
 # @TEST-EXEC: btest-bg-run send bro -b ../send.bro
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 #
 # @TEST-EXEC: btest-diff recv/.stdout
 # @TEST-EXEC: btest-diff send/.stdout
@@ -14,7 +14,7 @@
 @load base/frameworks/communication
 
 redef Communication::nodes += {
-    ["foo"] = [$host=[::1], $connect=T, $events=/my_event/]
+    ["foo"] = [$host=[::1], $connect=T, $retry=1sec, $events=/my_event/]
 };
 
 global my_event: event(s: string);

testing/btest/istate/broccoli-ipv6-socket.bro

@@ -4,7 +4,8 @@
 # @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1"
 #
 # @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ipv6=T"
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs -6 ::1
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff broccoli/.stdout

testing/btest/istate/broccoli-ipv6.bro

@@ -3,7 +3,8 @@
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
 # @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff broccoli/.stdout

testing/btest/istate/broccoli-ssl.bro

@@ -4,8 +4,9 @@
 #
 # @TEST-EXEC: chmod 600 broccoli.conf
 # @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ssl=T" "ssl_ca_certificate=../ca_cert.pem" "ssl_private_key=../bro.pem"
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli BROCCOLI_CONFIG_FILE=../broccoli.conf $BUILD/aux/broccoli/test/broccoli-v6addrs
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff broccoli/.stdout
 

testing/btest/istate/broccoli.bro

@@ -3,8 +3,9 @@
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
 # @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broping-record.bro
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broping -r -c 3 127.0.0.1
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: cat bro/ping.log | sed 's/one-way.*//g' >bro.log
 # @TEST-EXEC: cat broccoli/.stdout | sed 's/time=.*//g' >broccoli.log
 # @TEST-EXEC: btest-diff bro.log

testing/btest/istate/events-ssl.bro

@@ -2,7 +2,7 @@
 #
 # @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
@@ -55,7 +55,7 @@ event bro_init()
 redef peer_description = "events-rcv";
 
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T]
+    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T, $retry=1sec]
 };
 
 redef ssl_ca_certificate = "../ca_cert.pem";

testing/btest/istate/events.bro

@@ -2,7 +2,7 @@
 #
 # @TEST-EXEC: btest-bg-run sender   bro -Bthreading,logging,comm -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro -Bthreading,logging,comm  ../receiver.bro
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
@@ -50,7 +50,7 @@ event bro_init()
 redef peer_description = "events-rcv";
 
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T]
+    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $retry=1sec]
 };
 
 event remote_connection_closed(p: event_peer)

testing/btest/istate/sync.bro

@@ -154,7 +154,8 @@ event bro_init()
     }
 	
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /.*/, $connect=T, $sync=T]
+    ["foo"] = [$host = 127.0.0.1, $events = /.*/, $connect=T, $sync=T,
+               $retry=1sec]
 };
 
 event remote_connection_closed(p: event_peer)

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -8,9 +8,9 @@
 @TEST-START-FILE input.log
 #separator \x09
 #path	ssh
-#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	ns
-#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	string
-T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242
 @TEST-END-FILE
 
 @load base/protocols/ssh
@@ -37,6 +37,7 @@ type Val: record {
 	t: time;
 	iv: interval;
 	s: string;
+	ns: string;
 	sc: set[count];
 	ss: set[string];
 	se: set[string];
@@ -57,6 +58,7 @@ event bro_init()
 event Input::update_finished(name: string, source:string)
 	{
 	print outfile, servers;
+	print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail.
 	close(outfile);
 	terminate();
 	}