Merge remote-tracking branch 'origin/fastpath'

* origin/fastpath:
  Fix construction of ip6_ah (Authentication Header) record values.
  Update compile/dependency docs for OS X.
  Adjusting Mac binary packaging script.
  Unit test reliability adjustment.
  Adjusting some unit tests that do cluster communication.
  Small change to non-blocking DNS initialization.
  reorder a few statements in scan.l to make 1.5msecs etc work.

doc/quickstart.rst

@@ -1,5 +1,6 @@
 .. _CMake: http://www.cmake.org
 .. _SWIG: http://www.swig.org
+.. _Xcode: https://developer.apple.com/xcode/
 .. _MacPorts: http://www.macports.org
 .. _Fink: http://www.finkproject.org
 .. _Homebrew: http://mxcl.github.com/homebrew
@@ -85,17 +86,20 @@ The following dependencies are required to build Bro:
 
 * Mac OS X
 
-  Snow Leopard (10.6) comes with all required dependencies except for CMake_.
+  Compiling source code on Macs requires first downloading Xcode_,
+  then going through its "Preferences..." -> "Downloads" menus to
+  install the "Command Line Tools" component.
 
-  Lion (10.7) comes with all required dependencies except for CMake_ and SWIG_.
+  Lion (10.7) and Mountain Lion (10.8) come with all required
+  dependencies except for CMake_, SWIG_, and ``libmagic``.
 
-  Distributions of these dependencies can be obtained from the project websites
+  Distributions of these dependencies can be obtained from the project
-  linked above, but they're also likely available from your preferred Mac OS X
+  websites linked above, but they're also likely available from your
-  package management system (e.g. MacPorts_, Fink_, or Homebrew_).
+  preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
+  or Homebrew_).
 
-  Note that the MacPorts ``swig`` package may not include any specific
+  Specifically for MacPorts, the ``swig``, ``swig-ruby``, ``swig-python``
-  language support so you may need to also install ``swig-ruby`` and
+  and ``file`` packages provide the required dependencies.
-  ``swig-python``.
 
 Optional Dependencies
 ~~~~~~~~~~~~~~~~~~~~~

pkg/make-mac-packages

@@ -3,7 +3,13 @@
 # This script creates binary packages for Mac OS X.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
+cmake -P /dev/stdin << "EOF"
+if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
+    message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
+endif ()
+EOF
+
+[ $? -ne 0 ] && exit 1;
 
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
@@ -34,26 +40,26 @@ prefix=/opt/bro
 cd ..
 
 # Minimum Bro
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \
     --binary-package
 ( cd build && make package )
 
 # Full Bro package
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
 cd aux/broccoli
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..
 
 # Broctl
 cd aux/broctl
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..

scripts/base/init-bare.bro

@@ -1135,10 +1135,10 @@ type ip6_ah: record {
 	rsv: count;
 	## Security Parameter Index.
 	spi: count;
-	## Sequence number.
+	## Sequence number, unset in the case that *len* field is zero.
-	seq: count;
+	seq: count &optional;
-	## Authentication data.
+	## Authentication data, unset in the case that *len* field is zero.
-	data: string;
+	data: string &optional;
 };
 
 ## Values extracted from an IPv6 ESP extension header.

src/IP.cc

@@ -148,9 +148,15 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const
 		rv->Assign(1, new Val(((ip6_ext*)data)->ip6e_len, TYPE_COUNT));
 		rv->Assign(2, new Val(ntohs(((uint16*)data)[1]), TYPE_COUNT));
 		rv->Assign(3, new Val(ntohl(((uint32*)data)[1]), TYPE_COUNT));
-		rv->Assign(4, new Val(ntohl(((uint32*)data)[2]), TYPE_COUNT));
+
-		uint16 off = 3 * sizeof(uint32);
+		if ( Length() >= 12 )
-		rv->Assign(5, new StringVal(new BroString(data + off, Length() - off, 1)));
+			{
+			// Sequence Number and ICV fields can only be extracted if
+			// Payload Len was non-zero for this header.
+			rv->Assign(4, new Val(ntohl(((uint32*)data)[2]), TYPE_COUNT));
+			uint16 off = 3 * sizeof(uint32);
+			rv->Assign(5, new StringVal(new BroString(data + off, Length() - off, 1)));
+			}
 		}
 		break;
 

src/nb_dns.c

@@ -124,7 +124,7 @@ nb_dns_init(char *errstr)
 	nd->s = -1;
 
 	/* XXX should be able to init static hostent struct some other way */
-	(void)gethostbyname("localhost.");
+	(void)gethostbyname("localhost");
 
 	if ((_res.options & RES_INIT) == 0 && res_init() == -1) {
 		snprintf(errstr, NB_DNS_ERRSIZE, "res_init() failed");

src/scan.l

@@ -479,12 +479,6 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 		RET_CONST(new PortVal(p, TRANSPORT_UNKNOWN))
 		}
 
-({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
-
-"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
-
-{H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
-
 {FLOAT}{OWS}day(s?)	RET_CONST(new IntervalVal(atof(yytext),Days))
 {FLOAT}{OWS}hr(s?)	RET_CONST(new IntervalVal(atof(yytext),Hours))
 {FLOAT}{OWS}min(s?)	RET_CONST(new IntervalVal(atof(yytext),Minutes))
@@ -492,6 +486,12 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 {FLOAT}{OWS}msec(s?)	RET_CONST(new IntervalVal(atof(yytext),Milliseconds))
 {FLOAT}{OWS}usec(s?)	RET_CONST(new IntervalVal(atof(yytext),Microseconds))
 
+({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
+
+"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
+
+{H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
+
 \"([^\\\n\"]|{ESCSEQ})*\"	{
 	const char* text = yytext;
 	int len = strlen(text) + 1;

testing/btest/Baseline/core.ipv6_zero_len_ah/output

@@ -0,0 +1,2 @@
+[orig_h=2000:1300::1, orig_p=128/icmp, resp_h=2000:1300::2, resp_p=129/icmp]
+[ip=<uninitialized>, ip6=[class=0, flow=0, len=166, nxt=51, hlim=255, src=2000:1300::1, dst=2000:1300::2, exts=[[id=51, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=[nxt=58, len=0, rsv=0, spi=0, seq=<uninitialized>, data=<uninitialized>], esp=<uninitialized>, mobility=<uninitialized>]]], tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]

testing/btest/core/ipv6_zero_len_ah.test

@@ -0,0 +1,11 @@
+# @TEST-EXEC: bro -r $TRACES/ipv6_zero_len_ah.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# Shouldn't crash, but we also won't have seq and data fields set of the ip6_ah
+# record.
+
+event ipv6_ext_headers(c: connection, p: pkt_hdr)
+    {
+    print c$id;
+    print p;
+    }

testing/btest/scripts/base/frameworks/cluster/start-it-up.bro

@@ -1,11 +1,13 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff proxy-1/.stdout
 # @TEST-EXEC: btest-diff proxy-2/.stdout
@@ -22,7 +24,42 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+global fully_connected: event();
+
+global peer_count = 0;
+
+global fully_connected_nodes = 0;
+
+event fully_connected()
+	{
+	fully_connected_nodes = fully_connected_nodes + 1;
+	if ( Cluster::node == "manager-1" )
+		{
+		if ( peer_count == 4 && fully_connected_nodes == 4 )
+			terminate_communication();
+		}
+	}
+
+redef Cluster::worker2manager_events += /fully_connected/;
+redef Cluster::proxy2manager_events += /fully_connected/;
+
 event remote_connection_handshake_done(p: event_peer)
 	{
 	print "Connected to a peer";
+	peer_count = peer_count + 1;
+	if ( Cluster::node == "manager-1" )
+		{
+		if ( peer_count == 4 && fully_connected_nodes == 4 )
+			terminate_communication();
+		}
+	else
+		{
+		if ( peer_count == 2 )
+			event fully_connected();
+		}
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
 	}

testing/btest/scripts/base/frameworks/notice/cluster.bro

@@ -2,9 +2,9 @@
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -21,13 +21,44 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready: event();
+
+redef Cluster::manager2worker_events += /ready/;
+
 event delayed_notice()
 	{
 	if ( Cluster::node == "worker-1" )
 		NOTICE([$note=Test_Notice, $msg="test notice!"]);
 	}
 
-event bro_init()
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready()
 	{
 	schedule 1secs { delayed_notice() };
 	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	peer_count = peer_count + 1;
+	if ( peer_count == 2 )
+		event ready();
+	}
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	}
+
+@endif

testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro

@@ -2,10 +2,10 @@
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -23,6 +23,15 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready: event();
+
+redef Cluster::manager2worker_events += /ready/;
+
 event delayed_notice()
 	{
 	NOTICE([$note=Test_Notice,
@@ -30,10 +39,33 @@ event delayed_notice()
 	        $identifier="this identifier is static"]);
 	}
 
-event bro_init() &priority=5
+@if ( Cluster::local_node_type() == Cluster::WORKER )
-	{
+
+event ready()
+    {
 	if ( Cluster::node == "worker-1" )
 		schedule 4secs { delayed_notice() };
 	if ( Cluster::node == "worker-2" )
 		schedule 1secs { delayed_notice() };
+    }
+
+event Notice::suppressed(n: Notice::Info)
+	{
+	if ( Cluster::node == "worker-1" )
+		terminate_communication();
 	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		event ready();
+	}
+
+@endif