Merge branch 'master' of git://git.bro-ids.org/bro into elasticsearch

CHANGES

@@ -1,4 +1,55 @@
 
+2.0-608 | 2012-06-11 15:59:00 -0700
+
+  * Add more error handling code to logging of enum vals. Addresses
+    #829. (Jon Siwek)
+
+2.0-606 | 2012-06-11 15:55:56 -0700
+
+  * Fix summary lines for BIF documentation and corrected the
+    description of "fmt" and "floor" BIFs. (Daniel Thayer)
+
+  * Fix val_size BIF tests and improve docs. (Daniel Thayer)
+    
+2.0-602 | 2012-06-07 15:06:19 -0700
+
+  * Include header for usleep(), caused compile failure on Archlinux. (Jon Siwek)
+
+  * Revert "Fixed a bug with the MIME analyzer not removing whitespace
+    on wrapped headers." Needs discussion. (Robin Sommer)
+
+2.0-598 | 2012-06-06 11:47:00 -0700
+
+  * Add @load-sigs directive for loading signature files (addresses
+    #551). This can be used to load signatures relative to the current
+    scripts (e.g., "@load-sigs ./foo.sig"). (Jon Siwek)
+
+
+2.0-596 | 2012-06-06 11:41:00 -0700
+
+    * Fixes for some BiFs and their documentation. (Daniel Thayer)
+
+    * Many new unit tests for BiFs. (Daniel Thayer)
+
+2.0-579 | 2012-06-06 11:04:46 -0700
+
+  * Memory leak fixes for bad usages of VectorVal ctor. (Jon Siwek)
+
+  * Fixed a bug with the MIME analyzer not removing whitespace on
+    wrapped headers. (Seth Hall)
+
+  * Change Input::update_finished lookup to happen at init time. (Jon Siwek)
+
+  * Fix going through the internal_handler() function which will now
+    set the event as "used" (i.e. it's marked as being raised
+    somewhere). Addresses #823. (Jon Siwek)
+
+  * Fix format specifier on RemoteSerializer::Connect. This caused
+    32-bit systems to show a warning at compile-time, and fail when
+    connecting to peers. (Jon Siwek)
+
+  * Fixes for running tests in parallel. (Robin Sommer)
+
 2.0-571 | 2012-05-30 19:12:43 -0700
 
   * Updating submodule(s).

NEWS

@@ -55,6 +55,11 @@ Bro 2.1
   of that script, you need to adapt it. See the shipped version for
   details.
 
+- Signature files can now be loaded via the new "@load-sigs"
+  directive. In contrast to the existing (and still supported)
+  signature_files constant, this can be used to load signatures
+  relative to the current script (e.g., "@load-sigs ./foo.sig").
+
 TODO: Extend.
 
 Bro 2.0

VERSION

@@ -1 +1 @@
-2.0-571
+2.0-608

aux/binpac

@@ -1 +1 @@
-Subproject commit b4094cb75e0a7769123f7db1f5d73f3f9f1c3977
+Subproject commit 6f43a8115d8e6483a50957c5d21c5d69270ab3aa

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 2038e3de042115c3caa706426e16c830c1fd1e9e
+Subproject commit c6391412e902e896836450ab98910309b2ca2d9b

aux/broccoli

@@ -1 +1 @@
-Subproject commit 4e17842743fef8df6abf0588c7ca86c6937a2b6d
+Subproject commit 0d139c09d5a9c8623ecc2a5f395178f0ddcd7e16

aux/broctl

@@ -1 +1 @@
-Subproject commit 892b60edb967bb456872638f22ba994e84530137
+Subproject commit 880f3e48d33bb28d17184656f858a4a0e2e1574c

cmake

@@ -1 +1 @@
-Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b
+Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130

doc/signatures.rst

@@ -51,13 +51,18 @@ This script contains a default event handler that raises
 :bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
 (as well as others; see the beginning of the script).
 
-As signatures are independent of Bro's policy scripts, they are put
-into their own file(s). There are two ways to specify which files
-contain signatures: By using the ``-s`` flag when you invoke Bro, or
-by extending the Bro variable :bro:id:`signature_files` using the ``+=``
-operator. If a signature file is given without a path, it is searched
-along the normal ``BROPATH``. The default extension of the file name
-is ``.sig``, and Bro appends that automatically when necessary.
+As signatures are independent of Bro's policy scripts, they are put into
+their own file(s). There are three ways to specify which files contain
+signatures: By using the ``-s`` flag when you invoke Bro, or by
+extending the Bro variable :bro:id:`signature_files` using the ``+=``
+operator, or by using the ``@load-sigs`` directive inside a Bro script.
+If a signature file is given without a full path, it is searched for
+along the normal ``BROPATH``.  Additionally, the ``@load-sigs``
+directive can be used to load signature files in a path relative to the
+Bro script in which it's placed, e.g. ``@load-sigs ./mysigs.sig`` will
+expect that signature file in the same directory as the Bro script. The
+default extension of the file name is ``.sig``, and Bro appends that
+automatically when necessary.
 
 Signature language
 ==================

scripts/base/frameworks/dpd/main.bro

@@ -3,8 +3,7 @@
 
 module DPD;
 
-## Add the DPD signatures to the signature framework.
-redef signature_files += "base/frameworks/dpd/dpd.sig";
+@load-sigs ./dpd.sig
 
 export {
 	## Add the DPD logging stream identifier.

scripts/base/init-bare.bro

@@ -615,7 +615,9 @@ function add_signature_file(sold: string, snew: string): string
 	}
 
 ## Signature files to read. Use ``redef signature_files  += "foo.sig"`` to
-## extend. Signature files will be searched relative to ``BROPATH``.
+## extend. Signature files added this way will be searched relative to
+## ``BROPATH``.  Using the ``@load-sigs`` directive instead is preferred
+## since that can search paths relative to the current script.
 global signature_files = "" &add_func = add_signature_file;
 
 ## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``.
@@ -977,6 +979,9 @@ type ip6_option: record {
 	data: string;	##< Option data.
 };
 
+## A type alias for a vector of IPv6 options.
+type ip6_options: vector of ip6_option;
+
 ## Values extracted from an IPv6 Hop-by-Hop options extension header.
 ##
 ## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option
@@ -987,7 +992,7 @@ type ip6_hopopts: record {
 	## Length of header in 8-octet units, excluding first unit.
 	len: count;
 	## The TLV encoded options;
-	options: vector of ip6_option;
+	options: ip6_options;
 };
 
 ## Values extracted from an IPv6 Destination options extension header.
@@ -1000,7 +1005,7 @@ type ip6_dstopts: record {
 	## Length of header in 8-octet units, excluding first unit.
 	len: count;
 	## The TLV encoded options;
-	options: vector of ip6_option;
+	options: ip6_options;
 };
 
 ## Values extracted from an IPv6 Routing extension header.
@@ -1245,6 +1250,9 @@ type ip6_ext_hdr: record {
 	mobility: ip6_mobility_hdr &optional;
 };
 
+## A type alias for a vector of IPv6 extension headers
+type ip6_ext_hdr_chain: vector of ip6_ext_hdr;
+
 ## Values extracted from an IPv6 header.
 ##
 ## .. bro:see:: pkt_hdr ip4_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts
@@ -1259,7 +1267,7 @@ type ip6_hdr: record {
 	hlim: count;					##< Hop limit.
 	src: addr;					##< Source address.
 	dst: addr;					##< Destination address.
-	exts: vector of ip6_ext_hdr;			##< Extension header chain.
+	exts: ip6_ext_hdr_chain;			##< Extension header chain.
 };
 
 ## Values extracted from an IPv4 header.

scripts/base/protocols/http/file-ident.bro

@@ -6,7 +6,8 @@
 @load ./utils
 
 # Add the magic number signatures to the core signature set.
-redef signature_files += "base/protocols/http/file-ident.sig";
+@load-sigs ./file-ident.sig
+
 # Ignore the signatures used to match files
 redef Signatures::ignored_ids += /^matchfile-/;
 

scripts/policy/protocols/http/detect-webapps.bro

@@ -4,9 +4,10 @@
 @load base/frameworks/software
 @load base/protocols/http
 
+@load-sigs ./detect-webapps.sig
+
 module HTTP;
 
-redef signature_files += "protocols/http/detect-webapps.sig";
 # Ignore the signatures used to match webapps
 redef Signatures::ignored_ids += /^webapp-/;
 

scripts/site/local.bro

@@ -25,7 +25,7 @@ redef Software::vulnerable_versions += {
 @load frameworks/software/version-changes
 
 # This adds signatures to detect cleartext forward and reverse windows shells.
-redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
+@load-sigs frameworks/signatures/detect-windows-shells
 
 # Uncomment the following line to begin receiving (by default hourly) emails
 # containing all of your notices.

src/IP.cc

@@ -36,13 +36,12 @@ static inline RecordType* hdrType(RecordType*& type, const char* name)
 
 static VectorVal* BuildOptionsVal(const u_char* data, int len)
 	{
-	VectorVal* vv = new VectorVal(new VectorType(
-	        hdrType(ip6_option_type, "ip6_option")->Ref()));
+	VectorVal* vv = new VectorVal(internal_type("ip6_options")->AsVectorType());
 
 	while ( len > 0 )
 		{
 		const struct ip6_opt* opt = (const struct ip6_opt*) data;
-		RecordVal* rv = new RecordVal(ip6_option_type);
+		RecordVal* rv = new RecordVal(hdrType(ip6_option_type, "ip6_option"));
 		rv->Assign(0, new Val(opt->ip6o_type, TYPE_COUNT));
 
 		if ( opt->ip6o_type == 0 )
@@ -87,8 +86,8 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const
 		rv->Assign(5, new AddrVal(IPAddr(ip6->ip6_src)));
 		rv->Assign(6, new AddrVal(IPAddr(ip6->ip6_dst)));
 		if ( ! chain )
-			chain = new VectorVal(new VectorType(
-			        hdrType(ip6_ext_hdr_type, "ip6_ext_hdr")->Ref()));
+			chain = new VectorVal(
+			    internal_type("ip6_ext_hdr_chain")->AsVectorType());
 		rv->Assign(7, chain);
 		}
 		break;
@@ -583,7 +582,8 @@ VectorVal* IPv6_Hdr_Chain::BuildVal() const
 		ip6_mob_type = internal_type("ip6_mobility_hdr")->AsRecordType();
 		}
 
-	VectorVal* rval = new VectorVal(new VectorType(ip6_ext_hdr_type->Ref()));
+	VectorVal* rval = new VectorVal(
+	    internal_type("ip6_ext_hdr_chain")->AsVectorType());
 
 	for ( size_t i = 1; i < chain.size(); ++i )
 		{

src/Net.cc

@@ -69,6 +69,7 @@ PktSrc* current_pktsrc = 0;
 IOSource* current_iosrc;
 
 std::list<ScannedFile> files_scanned;
+std::vector<string> sig_files;
 
 RETSIGTYPE watchdog(int /* signo */)
 	{

src/Net.h

@@ -111,5 +111,6 @@ struct ScannedFile {
 };
 
 extern std::list<ScannedFile> files_scanned;
+extern std::vector<string> sig_files;
 
 #endif

src/Type.cc

@@ -1467,6 +1467,16 @@ bool VectorType::DoUnserialize(UnserialInfo* info)
 	return yield_type != 0;
 	}
 
+void VectorType::Describe(ODesc* d) const
+	{
+	if ( d->IsReadable() )
+		d->AddSP("vector of");
+	else
+		d->Add(int(Tag()));
+
+	yield_type->Describe(d);
+	}
+
 BroType* base_type(TypeTag tag)
 	{
 	static BroType* base_types[NUM_TYPES];

src/Type.h

@@ -564,6 +564,8 @@ public:
 	// gets using an empty "vector()" constructor.
 	bool IsUnspecifiedVector() const;
 
+	void Describe(ODesc* d) const;
+
 protected:
 	VectorType()	{ yield_type = 0; }
 

src/bro.bif

@@ -466,17 +466,18 @@ function system%(str: string%): int
 ##
 ## str: The command to execute.
 ##
-## env: A :bro:type:`set` or :bro:type:`table` with the environment variables
-##      in the form of key-value pairs (where the value is optional).
+## env: A :bro:type:`table` with the environment variables in the form
+##      of key-value pairs. Each specified environment variable name
+##      will be automatically prepended with ``BRO_ARG_``.
 ##
 ## Returns: The return value from the OS ``system`` function.
 ##
 ## .. bro:see:: system str_shell_escape piped_exec
-function system_env%(str: string, env: any%): int
+function system_env%(str: string, env: table_string_of_string%): int
 	%{
 	if ( env->Type()->Tag() != TYPE_TABLE )
 		{
-		builtin_error("system_env() requires a table/set argument");
+		builtin_error("system_env() requires a table argument");
 		return new Val(-1, TYPE_INT);
 		}
 
@@ -1422,12 +1423,15 @@ bool indirect_int_sort_function(int a, int b)
 
 ## Sorts a vector in place. The second argument is a comparison function that
 ## takes two arguments: if the vector type is ``vector of T``, then the
-## comparison function must be ``function(a: T, b: T): bool``, which returns
-## ``a < b`` for some type-specific notion of the less-than operator.
+## comparison function must be ``function(a: T, b: T): int``, which returns
+## a value less than zero if ``a < b`` for some type-specific notion of the
+## less-than operator.  The comparison function is optional if the type
+## is an integral type (int, count, etc.).
 ##
 ## v: The vector instance to sort.
 ##
-## Returns: The original vector.
+## Returns: The vector, sorted from minimum to maximum value. If the vector
+##          could not be sorted, then the original vector is returned instead.
 ##
 ## .. bro:see:: order
 function sort%(v: any, ...%) : any
@@ -1490,12 +1494,14 @@ function sort%(v: any, ...%) : any
 ## v: The vector whose order to compute.
 ##
 ## Returns: A ``vector of count`` with the indices of the ordered elements.
+##          For example, the elements of *v* in order are (assuming ``o``
+##          is the vector returned by ``order``):  v[o[0]], v[o[1]], etc.
 ##
 ## .. bro:see:: sort
 function order%(v: any, ...%) : index_vec
 	%{
-	VectorVal* result_v =
-		new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+	VectorVal* result_v = new VectorVal(
+	    internal_type("index_vec")->AsVectorType());
 
 	if ( v->Type()->Tag() != TYPE_VECTOR )
 		{
@@ -1522,13 +1528,13 @@ function order%(v: any, ...%) : index_vec
 		}
 
 	if ( ! comp && ! IsIntegral(elt_type->Tag()) )
-		builtin_error("comparison function required for sort() with non-integral types");
+		builtin_error("comparison function required for order() with non-integral types");
 
 	vector<Val*>& vv = *v->AsVector();
 	int n = vv.size();
 
 	// Set up initial mapping of indices directly to corresponding
-	// elements.  We stay zero-based until after the sorting.
+	// elements.
 	vector<int> ind_vv(n);
 	index_map = new Val*[n];
 	int i;
@@ -1544,7 +1550,7 @@ function order%(v: any, ...%) : index_vec
 		if ( comp_type->YieldType()->Tag() != TYPE_INT ||
 		     ! comp_type->ArgTypes()->AllMatch(elt_type, 0) )
 			{
-			builtin_error("invalid comparison function in call to sort()");
+			builtin_error("invalid comparison function in call to order()");
 			return v;
 			}
 
@@ -1558,8 +1564,7 @@ function order%(v: any, ...%) : index_vec
 	delete [] index_map;
 	index_map = 0;
 
-	// Now spin through ind_vv to read out the rearrangement,
-	// adjusting indices as we do so.
+	// Now spin through ind_vv to read out the rearrangement.
 	for ( i = 0; i < n; ++i )
 		{
 		int ind = ind_vv[i];
@@ -1649,7 +1654,7 @@ function cat_sep%(sep: string, def: string, ...%): string
 ##
 ##    - ``.``: Precision of floating point specifiers ``[efg]`` (< 128)
 ##
-##    - ``A``: Escape NUL bytes, i.e., replace ``0`` with ``\0``
+##    - ``A``: Escape only NUL bytes (each one replaced with ``\0``) in a string
 ##
 ##    - ``[DTdxsefg]``: Format specifier
 ##
@@ -1661,13 +1666,13 @@ function cat_sep%(sep: string, def: string, ...%): string
 ##        - ``x``: Unsigned hexadecimal (using C-style ``%llx``);
 ##                 addresses/ports are converted to host-byte order
 ##
-##        - ``s``: Escaped string
+##        - ``s``: String (byte values less than 32 or greater than 126
+##                 will be escaped)
 ##
 ##        - ``[efg]``: Double
 ##
-## Returns: Given no arguments, :bro:id:`fmt` returns an empty string. Given a
-##          non-string first argument, :bro:id:`fmt` returns the concatenation
-##          of all its arguments, per :bro:id:`cat`. Finally, given the wrong
+## Returns: Returns the formatted string. Given no arguments, :bro:id:`fmt`
+##          returns an empty string. Given no format string or the wrong
 ##          number of additional arguments for the given format specifier,
 ##          :bro:id:`fmt` generates a run-time error.
 ##
@@ -1678,8 +1683,9 @@ function fmt%(...%): string
 		return new StringVal("");
 
 	Val* fmt_v = @ARG@[0];
-	if ( fmt_v->Type()->Tag() != TYPE_STRING )
-		return bro_cat(frame, @ARGS@);
+
+	// Type of fmt_v will be string here, check_built_in_call() in Func.cc
+	// checks that.
 
 	const char* fmt = fmt_v->AsString()->CheckString();
 	ODesc d;
@@ -1689,10 +1695,16 @@ function fmt%(...%): string
 		;
 
 	if ( n < @ARGC@ - 1 )
+		{
 		builtin_error("too many arguments for format", fmt_v);
+		return new StringVal("");
+		}
 
 	else if ( n >= @ARGC@ ) 
+		{
 		builtin_error("too few arguments for format", fmt_v);
+		return new StringVal("");
+		}
 
 	BroString* s = new BroString(1, d.TakeBytes(), d.Len());
 	s->SetUseFreeToDelete(true);
@@ -1706,8 +1718,9 @@ function fmt%(...%): string
 #
 # ===========================================================================
 
-## Chops off any decimal digits of the given double, i.e., computes the
-## "floor" of it. For example, ``floor(3.14)`` returns ``3.0``.
+## Computes the greatest integer less than the given :bro:type:`double` value.
+## For example, ``floor(3.14)`` returns ``3.0``, and ``floor(-3.14)``
+## returns ``-4.0``.
 ##
 ## d: The :bro:type:`double` to manipulate.
 ##
@@ -1886,8 +1899,9 @@ function reading_traces%(%): bool
 	return new Val(reading_traces, TYPE_BOOL);
 	%}
 
-## Returns statistics about the number of packets *(i)* received by Bro,
-## *(ii)* dropped, and *(iii)* seen on the link (not always available).
+## Returns packet capture statistics. Statistics include the number of
+## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the
+## link (not always available).
 ##
 ## Returns: A record of packet statistics.
 ##
@@ -1921,9 +1935,9 @@ function net_stats%(%): NetStats
 	return ns;
 	%}
 
-## Returns Bro process statistics, such as real/user/sys CPU time, memory
-## usage, page faults, number of TCP/UDP/ICMP connections, timers, and events
-## queued/dispatched.
+## Returns Bro process statistics. Statistics include real/user/sys CPU time,
+## memory usage, page faults, number of TCP/UDP/ICMP connections, timers,
+## and events queued/dispatched.
 ##
 ## Returns: A record with resource usage statistics.
 ##
@@ -1998,10 +2012,10 @@ function resource_usage%(%): bro_resources
 	return res;
 	%}
 
-## Returns statistics about the regular expression engine, such as the number
-## of distinct matchers, DFA states, DFA state transitions, memory usage of
-## DFA states, cache hits/misses, and average number of NFA states across all
-## matchers.
+## Returns statistics about the regular expression engine. Statistics include
+## the number of distinct matchers, DFA states, DFA state transitions, memory
+## usage of DFA states, cache hits/misses, and average number of NFA states
+## across all matchers.
 ##
 ## Returns: A record with matcher statistics.
 ##
@@ -2181,10 +2195,10 @@ function record_fields%(rec: any%): record_field_table
 	return fields;
 	%}
 
-## Enables detailed collections of statistics about CPU/memory usage,
-## connections, TCP states/reassembler, DNS lookups, timers, and script-level
-## state. The script variable :bro:id:`profiling_file` holds the name of the
-## file.
+## Enables detailed collection of profiling statistics. Statistics include
+## CPU/memory usage, connections, TCP states/reassembler, DNS lookups,
+## timers, and script-level state. The script variable :bro:id:`profiling_file`
+## holds the name of the file.
 ##
 ## .. bro:see:: net_stats
 ##              resource_usage
@@ -2331,7 +2345,7 @@ function is_v6_addr%(a: addr%): bool
 ## Returns: The vector of addresses contained in the routing header data.
 function routing0_data_to_addrs%(s: string%): addr_vec
 	%{
-	VectorVal* rval = new VectorVal(new VectorType(base_type(TYPE_ADDR)));
+	VectorVal* rval = new VectorVal(internal_type("addr_vec")->AsVectorType());
 
 	int len = s->Len();
 	const u_char* bytes = s->Bytes();
@@ -2362,7 +2376,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec
 ## .. bro:see:: counts_to_addr
 function addr_to_counts%(a: addr%): index_vec
 	%{
-	VectorVal* rval = new VectorVal(new VectorType(base_type(TYPE_COUNT)));
+	VectorVal* rval = new VectorVal(internal_type("index_vec")->AsVectorType());
 	const uint32* bytes;
 	int len = a->AsAddr().GetBytes(&bytes);
 
@@ -2608,7 +2622,7 @@ function count_to_v4_addr%(ip: count%): addr
 	return new AddrVal(htonl(uint32(ip)));
 	%}
 
-## Converts a :bro:type:`string` of bytes into an IP address. In particular,
+## Converts a :bro:type:`string` of bytes into an IPv4 address. In particular,
 ## this function interprets the first 4 bytes of the string as an IPv4 address
 ## in network order.
 ##
@@ -2670,7 +2684,7 @@ function to_port%(s: string%): port
 ##
 ## Returns: The IP address corresponding to *s*.
 ##
-## .. bro:see:: addr_to_ptr_name parse_dotted_addr
+## .. bro:see:: addr_to_ptr_name to_addr
 function ptr_name_to_addr%(s: string%): addr
 	%{
 	if ( s->Len() != 72 )
@@ -2734,27 +2748,12 @@ function ptr_name_to_addr%(s: string%): addr
 ##
 ## Returns: The reverse pointer representation of *a*.
 ##
-## .. bro:see:: ptr_name_to_addr parse_dotted_addr
+## .. bro:see:: ptr_name_to_addr to_addr
 function addr_to_ptr_name%(a: addr%): string
 	%{
 	return new StringVal(a->AsAddr().PtrName().c_str());
 	%}
 
-# Transforms n0.n1.n2.n3 -> addr.
-
-## Converts a decimal dotted IP address in a :bro:type:`string` to an
-## :bro:type:`addr` type.
-##
-## s: The IP address in the form ``n0.n1.n2.n3``.
-##
-## Returns: The IP address as type :bro:type:`addr`.
-##
-## .. bro:see:: addr_to_ptr_name parse_dotted_addr
-function parse_dotted_addr%(s: string%): addr
-	%{
-	IPAddr a(s->CheckString());
-	return new AddrVal(a);
-	%}
 
 %%{
 static Val* parse_port(const char* line)
@@ -3022,8 +3021,8 @@ function decode_netbios_name_type%(name: string%): count
 	return new Val(return_val, TYPE_COUNT);
 	%}
 
-## Converts a string of bytes into its hexadecimal representation, e.g.,
-## ``"04"`` to ``"3034"``.
+## Converts a string of bytes into its hexadecimal representation.
+## For example, ``"04"`` would be converted to ``"3034"``.
 ##
 ## bytestring: The string of bytes.
 ##
@@ -3285,7 +3284,7 @@ function mask_addr%(a: addr, top_bits_to_keep: count%): subnet
 	return new SubNetVal(a->AsAddr(), top_bits_to_keep);
 	%}
 
-## Takes some top bits (e.g., subnet address) from one address and the other
+## Takes some top bits (such as a subnet address) from one address and the other
 ## bits (intra-subnet part) from a second address and merges them to get a new
 ## address. This is useful for anonymizing at subnet level while preserving
 ## serial scans.
@@ -3522,7 +3521,7 @@ function skip_http_entity_data%(c: connection, is_orig: bool%): any
 	return 0;
 	%}
 
-## Unescapes all characters in a URI, i.e., decodes every ``%xx`` group.
+## Unescapes all characters in a URI (decode every ``%xx`` group).
 ##
 ## URI: The URI to unescape.
 ##
@@ -5659,6 +5658,14 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
 #
 # ===========================================================================
 
+## Deprecated. Will be removed.
+function parse_dotted_addr%(s: string%): addr
+	%{
+	IPAddr a(s->CheckString());
+	return new AddrVal(a);
+	%}
+
+
 %%{
 #include "Anon.h"
 %%}

src/logging/Manager.cc

@@ -828,7 +828,13 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 		const char* s =
 			val->Type()->AsEnumType()->Lookup(val->InternalInt());
 
+		if ( s )
 			lval->val.string_val = new string(s);
+		else
+			{
+			val->Type()->Error("enum type does not contain value", val);
+			lval->val.string_val = new string();
+			}
 		break;
 		}
 

src/main.cc

@@ -838,6 +838,10 @@ int main(int argc, char** argv)
 		if ( *s )
 			rule_files.append(s);
 
+	// Append signature files defined in @load-sigs
+	for ( size_t i = 0; i < sig_files.size(); ++i )
+		rule_files.append(copy_string(sig_files[i].c_str()));
+
 	if ( rule_files.length() > 0 )
 		{
 		rule_matcher = new RuleMatcher(RE_level);

src/scan.l

@@ -358,6 +358,22 @@ when	return TOK_WHEN;
 	(void) load_files(new_file);
 	}
 
+@load-sigs{WS}{FILE} {
+	const char* new_sig_file = skip_whitespace(yytext + 10);
+	const char* full_filename = 0;
+	FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0);
+
+	if ( f )
+		{
+		sig_files.push_back(full_filename);
+		fclose(f);
+		delete [] full_filename;
+		}
+	else
+		reporter->Error("failed to find file associated with @load-sigs %s",
+		                new_sig_file);
+	}
+
 @unload{WS}{FILE}	{
 	// Skip "@unload".
 	const char* new_file = skip_whitespace(yytext + 7);

src/strings.bif

@@ -175,7 +175,7 @@ function join_string_vec%(vec: string_vec, sep: string%): string
 		if ( i > 0 )
 			d.Add(sep->CheckString(), 0);
 
-		v->Lookup(i+1)->Describe(&d);
+		v->Lookup(i)->Describe(&d);
 		}
 
 	BroString* s = new BroString(1, d.TakeBytes(), d.Len());
@@ -875,8 +875,8 @@ function str_split%(s: string, idx: index_vec%): string_vec
 		indices[i] = (*idx_v)[i]->AsCount();
 
 	BroString::Vec* result = s->AsString()->Split(indices);
-	VectorVal* result_v =
-		new VectorVal(new VectorType(base_type(TYPE_STRING)));
+	VectorVal* result_v = new VectorVal(
+	    internal_type("string_vec")->AsVectorType());
 
 	if ( result )
 		{

src/threading/MsgThread.cc

@@ -4,6 +4,8 @@
 #include "MsgThread.h"
 #include "Manager.h"
 
+#include <unistd.h>
+
 using namespace threading;
 
 namespace threading  {

testing/btest/Baseline/bifs.all_set/out

@@ -0,0 +1,3 @@
+F
+F
+T

testing/btest/Baseline/bifs.any_set/out

@@ -0,0 +1,3 @@
+T
+F
+F

testing/btest/Baseline/bifs.byte_len/out

@@ -0,0 +1 @@
+11

testing/btest/Baseline/bifs.bytestring_to_hexstr/out

@@ -0,0 +1,3 @@
+3034
+
+00

testing/btest/Baseline/bifs.cat/out

@@ -0,0 +1,6 @@
+foo3T
+
+3T
+foo|3|T
+
+<empty>|3|T

testing/btest/Baseline/bifs.cat_string_array/out

@@ -0,0 +1,3 @@
+isatest
+thisisatest
+isa

testing/btest/Baseline/bifs.clear_table/out

@@ -0,0 +1,2 @@
+1
+0

testing/btest/Baseline/bifs.convert_for_pattern/out

@@ -0,0 +1,3 @@
+foo
+
+b\[a\-z\]\+

testing/btest/Baseline/bifs.create_file/out

@@ -0,0 +1,15 @@
+T
+testfile
+F
+15.0
+T
+F
+28.0
+-1.0
+15.0
+0.0
+T
+15.0
+T
+testdir/testfile4
+F

testing/btest/Baseline/bifs.create_file/testfile

@@ -0,0 +1,2 @@
+This is a test
+another test

testing/btest/Baseline/bifs.create_file/testfile2

@@ -0,0 +1 @@
+new text

testing/btest/Baseline/bifs.edit/out

@@ -0,0 +1 @@
+llo t

testing/btest/Baseline/bifs.escape_string/out

@@ -0,0 +1,10 @@
+12
+Test \0string
+13
+Test \0string
+15
+Test \x00string
+13
+Test \0string
+24
+546573742000737472696e67

testing/btest/Baseline/bifs.exit/out

@@ -0,0 +1 @@
+hello

testing/btest/Baseline/bifs.file_mode/out

@@ -0,0 +1,10 @@
+rw-r--r--
+rwxrwxrwx
+rwxrwxrwt
+rwxr-x--T
+rwsr-xr-x
+r-S------
+rwxr-sr-x
+r--r-S---
+--xr-xrwx
+---------

testing/btest/Baseline/bifs.find_all/out

@@ -0,0 +1,4 @@
+es
+hi
+-------------------
+0

testing/btest/Baseline/bifs.find_entropy/out

@@ -0,0 +1,2 @@
+[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
+[entropy=2.083189, chi_square=3906.018182, mean=69.054545, monte_carlo_pi=4.0, serial_correlation=0.849402]

testing/btest/Baseline/bifs.find_last/out

@@ -0,0 +1,3 @@
+es
+-------------------
+0

testing/btest/Baseline/bifs.fmt/out

@@ -0,0 +1,55 @@
+test
+%
+
+*test      *
+*      test*
+*         T*
+*T         *
+*  3.14e+00*
+*3.14e+00  *
+*      3.14*
+*       3.1*
+* -3.14e+00*
+*     -3.14*
+*      -3.1*
+*-3.14e+00 *
+*-3.14     *
+*-3.1      *
+*      -128*
+*-128      *
+*       128*
+*0000000128*
+*128       *
+*        a0*
+*00000000a0*
+*        a0*
+*   160/tcp*
+* 127.0.0.1*
+*  7f000001*
+*192.168.0.0/16*
+*       ::1*
+*fe000000000000000000000000000001*
+*fe80:1234::1*
+*fe80:1234::/32*
+*   3.0 hrs*
+*/^?(^foo|bar)$?/*
+*      Blue*
+* [1, 2, 3]*
+*{^J^I2,^J^I1,^J^I3^J}*
+*{^J^I[2] = bro,^J^I[1] = test^J}*
+3.100000e+02
+310.000000
+310
+3.100e+02
+310.000
+310
+310
+2
+3
+4
+2
+2
+6
+2
+2
+6

testing/btest/Baseline/bifs.fmt_ftp_port/out

@@ -0,0 +1,2 @@
+192,168,0,2,1,1
+

testing/btest/Baseline/bifs.get_port_transport_proto/out

@@ -0,0 +1,3 @@
+tcp
+udp
+icmp

testing/btest/Baseline/bifs.getsetenv/out

@@ -0,0 +1,3 @@
+OK
+OK
+OK

testing/btest/Baseline/bifs.global_ids/out

@@ -0,0 +1 @@
+func

testing/btest/Baseline/bifs.hexdump/out

@@ -0,0 +1 @@
+0000  61 62 63 ff 64 65 66 67  68 69 6a 6b 6c 6d 6e 6f  abc.defg hijklmno^J0010  70 71 72 73 74 75 76 77  78 79 7a                 pqrstuvw xyz^J

testing/btest/Baseline/bifs.is_ascii/out

@@ -0,0 +1,2 @@
+F
+T

testing/btest/Baseline/bifs.is_port/out

@@ -0,0 +1,9 @@
+T
+F
+F
+F
+T
+F
+F
+F
+T

testing/btest/Baseline/bifs.join_string/out

@@ -0,0 +1,6 @@
+this * is * a * test
+thisisatest
+mytest
+this__is__another__test
+thisisanothertest
+Test

testing/btest/Baseline/bifs.length/out

@@ -0,0 +1,6 @@
+1
+4
+2
+0
+0
+0

testing/btest/Baseline/bifs.lookup_ID/out

@@ -0,0 +1,5 @@
+bro test
+<unknown id>
+<unknown id>
+<unknown id>
+event()

testing/btest/Baseline/bifs.lowerupper/out

@@ -0,0 +1,2 @@
+this is a test
+THIS IS A TEST

testing/btest/Baseline/bifs.math/out

@@ -0,0 +1,8 @@
+3.0
+2.0
+-4.0
+-3.0
+1.772005
+23.103867
+1.144223
+0.49693

testing/btest/Baseline/bifs.md5/output

@@ -2,3 +2,5 @@ f97c5d29941bfb1b2fdab0874906ab82
 7b0391feb2e0cd271f1cf39aafb4376f
 f97c5d29941bfb1b2fdab0874906ab82
 7b0391feb2e0cd271f1cf39aafb4376f
+571c0a35c7858ad5a0e16b8fdb41adcd
+1751cbd623726f423f734e23a8c7ec06

testing/btest/Baseline/bifs.merge_pattern/out

@@ -0,0 +1,2 @@
+match
+match

testing/btest/Baseline/bifs.order/out

@@ -0,0 +1,8 @@
+[5, 2, 8, 3]
+[1, 3, 0, 2]
+[5.0 hrs, 2.0 days, 1.0 sec, -7.0 mins]
+[3, 2, 0, 1]
+[192.168.123.200, 10.0.0.157, 192.168.0.3]
+[1, 2, 0]
+[3.03, 3.01, 3.02, 3.015]
+[1, 3, 2, 0]

testing/btest/Baseline/bifs.parse_ftp/out

@@ -0,0 +1,5 @@
+[h=192.168.0.2, p=257/tcp, valid=T]
+[h=192.168.0.2, p=257/tcp, valid=T]
+[h=fe80::12, p=1234/tcp, valid=T]
+[h=192.168.0.2, p=257/tcp, valid=T]
+[h=::, p=1234/tcp, valid=T]

testing/btest/Baseline/bifs.rand/out

@@ -0,0 +1,6 @@
+185
+236
+805
+47
+996
+498

testing/btest/Baseline/bifs.raw_bytes_to_v4_addr/out

@@ -0,0 +1,2 @@
+65.66.67.68
+0.0.0.0

testing/btest/Baseline/bifs.record_type_to_vector/out

@@ -0,0 +1 @@
+[, ct, str1]

testing/btest/Baseline/bifs.resize/out

@@ -0,0 +1,4 @@
+3
+5
+0
+7

testing/btest/Baseline/bifs.rotate_file/out

@@ -0,0 +1,3 @@
+file rotated
+15.0
+0.0

testing/btest/Baseline/bifs.rotate_file_by_name/out

@@ -0,0 +1,3 @@
+file rotated
+15.0
+0.0

testing/btest/Baseline/bifs.same_object/out

@@ -0,0 +1,3 @@
+T
+F
+F

testing/btest/Baseline/bifs.sort/out

@@ -0,0 +1,16 @@
+[2, 3, 5, 8]
+[2, 3, 5, 8]
+[-7.0 mins, 1.0 sec, 5.0 hrs, 2.0 days]
+[-7.0 mins, 1.0 sec, 5.0 hrs, 2.0 days]
+[F, F, T, T]
+[F, F, T, T]
+[57/tcp, 123/tcp, 7/udp, 500/udp, 12/icmp]
+[57/tcp, 123/tcp, 7/udp, 500/udp, 12/icmp]
+[3.03, 3.01, 3.02, 3.015]
+[3.03, 3.01, 3.02, 3.015]
+[192.168.123.200, 10.0.0.157, 192.168.0.3]
+[192.168.123.200, 10.0.0.157, 192.168.0.3]
+[10.0.0.157, 192.168.0.3, 192.168.123.200]
+[10.0.0.157, 192.168.0.3, 192.168.123.200]
+[3.01, 3.015, 3.02, 3.03]
+[3.01, 3.015, 3.02, 3.03]

testing/btest/Baseline/bifs.sort_string_array/out

@@ -0,0 +1,4 @@
+a
+is
+test
+this

testing/btest/Baseline/bifs.split/out

@@ -0,0 +1,32 @@
+t
+s is a t
+t
+---------------------
+t
+s is a test
+---------------------
+t
+hi
+s is a t
+es
+t
+---------------------
+t
+s is a test
+---------------------
+t
+hi
+s is a test
+---------------------
+[, thi, s i, s a tes, t]
+---------------------
+X-Mailer
+Testing Test (http://www.example.com)
+---------------------
+A 
+=
+ B 
+=
+ C 
+=
+ D

testing/btest/Baseline/bifs.str_shell_escape/out

@@ -0,0 +1,4 @@
+24
+echo ${TEST} > "my file"
+27
+echo \${TEST} > \"my file\"

testing/btest/Baseline/bifs.strcmp/out

@@ -0,0 +1,3 @@
+T
+T
+T

testing/btest/Baseline/bifs.string_fill/out

@@ -0,0 +1,3 @@
+*\0* 1
+*t\0* 2
+*test test\0* 10

testing/btest/Baseline/bifs.string_splitting/out

@@ -1,13 +0,0 @@
-{
-[2] = Testing Test (http://www.example.com),
-[1] = X-Mailer
-}
-{
-[2] = =,
-[4] = =,
-[6] = =,
-[7] =  D,
-[1] = A ,
-[5] =  C ,
-[3] =  B 
-}

testing/btest/Baseline/bifs.string_to_pattern/out

@@ -0,0 +1,6 @@
+/^?(foo)$?/
+/^?()$?/
+/^?(b[a-z]+)$?/
+/^?(foo)$?/
+/^?()$?/
+/^?(b\[a\-z\]\+)$?/

testing/btest/Baseline/bifs.strip/out

@@ -0,0 +1,6 @@
+*  this is a test *
+*this is a test*
+**
+**
+* *
+**

testing/btest/Baseline/bifs.strstr/out

@@ -0,0 +1,2 @@
+2
+0

testing/btest/Baseline/bifs.sub/out

@@ -0,0 +1,2 @@
+that is a test
+that at a test

testing/btest/Baseline/bifs.subst_string/out

@@ -0,0 +1 @@
+that at another test

testing/btest/Baseline/bifs.system/out

@@ -0,0 +1 @@
+thistest

testing/btest/Baseline/bifs.system_env/testfile

@@ -0,0 +1 @@
+helloworld

testing/btest/Baseline/bifs.to_count/out

@@ -0,0 +1,9 @@
+0
+2
+3
+4
+7
+0
+18446744073709551611
+0
+123

testing/btest/Baseline/bifs.to_double/out

@@ -0,0 +1,6 @@
+0.000001
+1.0
+-60.0
+3600.0
+86400.0
+1337982322.762159

testing/btest/Baseline/bifs.to_int/out

@@ -0,0 +1,3 @@
+1
+-1
+0

testing/btest/Baseline/bifs.to_interval/out

@@ -0,0 +1,2 @@
+1234563.14
+-1234563.14

testing/btest/Baseline/bifs.to_port/out

@@ -0,0 +1,7 @@
+123/tcp
+123/udp
+123/icmp
+0/unknown
+256/tcp
+256/udp
+256/icmp

testing/btest/Baseline/bifs.to_time/out

@@ -0,0 +1,2 @@
+1234563.14
+-1234563.14

testing/btest/Baseline/bifs.type_name/out

@@ -0,0 +1,26 @@
+string
+count
+int
+double
+bool
+time
+interval
+pattern
+enum
+port
+addr
+addr
+subnet
+subnet
+vector of count
+vector of table[count] of string
+set[count]
+set[port,string]
+table[count] of string
+table[string] of table[addr,port] of string
+record { c:count; s:string; }
+function(aa:int; bb:int;) : bool
+function() : any
+function() : void
+file of string
+event()

testing/btest/Baseline/bifs.uuid_to_string/out

@@ -0,0 +1,2 @@
+626180fe-6463-6665-6730-313233343536
+<Invalid UUID>

testing/btest/Baseline/core.leaks.ipv6_ext_headers/output

@@ -0,0 +1,4 @@
+weird routing0_hdr from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:78:1:32::2
+[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=53/udp, resp_h=2001:78:1:32::2, resp_p=53/udp]
+[ip=<uninitialized>, ip6=[class=0, flow=0, len=59, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>, mobility=<uninitialized>], [id=43, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=[nxt=17, len=4, rtype=0, segleft=2, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>, mobility=<uninitialized>]]], tcp=<uninitialized>, udp=[sport=53/udp, dport=53/udp, ulen=11], icmp=<uninitialized>]
+[2001:78:1:32::1, 2001:78:1:32::2]

testing/btest/Baseline/core.leaks.vector-val-bifs/output

@@ -0,0 +1,10 @@
+[1, 3, 0, 2]
+[2374950123]
+[1, 3, 0, 2]
+[2374950123]
+[1, 3, 0, 2]
+[2374950123]
+[1, 3, 0, 2]
+[3353991673]
+[1, 3, 0, 2]
+[3353991673]

testing/btest/Baseline/core.load-sigs/output

@@ -0,0 +1,3 @@
+[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp]
+works
+GET /images/wikimedia-button.png HTTP/1.1^M^JHost: meta.wikimedia.org^M^JUser-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Geck...

testing/btest/bifs/all_set.bro

@@ -0,0 +1,15 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = vector( T, F, T );
+	print all_set(a);
+
+	local b = vector();
+	print all_set(b);
+
+	local c = vector( T );
+	print all_set(c);
+	}

testing/btest/bifs/any_set.bro

@@ -0,0 +1,15 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = vector( F, T, F );
+	print any_set(a);
+
+	local b = vector();
+	print any_set(b);
+
+	local c = vector( F );
+	print any_set(c);
+	}

testing/btest/bifs/byte_len.bro

@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = "hello\0there";
+
+	print byte_len(a);
+	}

testing/btest/bifs/bytestring_to_hexstr.bro

@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	print bytestring_to_hexstr("04");
+	print bytestring_to_hexstr("");
+	print bytestring_to_hexstr("\0");
+	}

testing/btest/bifs/cat.bro

@@ -0,0 +1,22 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = "foo";
+	local b = 3;
+	local c = T;
+
+	print cat(a, b, c);
+
+	print cat();
+
+	print cat("", 3, T);
+
+	print cat_sep("|", "<empty>", a, b, c);
+
+	print cat_sep("|", "<empty>");
+	
+	print cat_sep("|", "<empty>", "", b, c);
+	}

testing/btest/bifs/cat_string_array.bro

@@ -0,0 +1,14 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a: string_array = { 
+		[0] = "this", [1] = "is", [2] = "a", [3] = "test" 
+	};
+
+	print cat_string_array(a);
+	print cat_string_array_n(a, 0, |a|-1);
+	print cat_string_array_n(a, 1, 2);
+	}

testing/btest/bifs/clear_table.bro

@@ -0,0 +1,14 @@
+#
+# @TEST-EXEC: bro %INPUT > out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local mytable: table[string] of string = { ["key1"] = "val1" };
+
+	print |mytable|;
+
+	clear_table(mytable);
+
+	print |mytable|;
+	}

testing/btest/bifs/convert_for_pattern.bro

@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	print convert_for_pattern("foo");
+	print convert_for_pattern("");
+	print convert_for_pattern("b[a-z]+");
+	}