Merge branch 'master' into topic/script-reference

CHANGES

@@ -1,4 +1,31 @@
 
+2.0-beta-98 | 2011-12-07 08:12:08 -0800
+
+  * Adapting test-suite's diff-all so that it expands globs in both
+    current and baseline directory. Closes #677. (Robin Sommer)
+
+2.0-beta-97 | 2011-12-06 11:49:29 -0800
+
+  * Omit loading local-<node>.bro scripts from base cluster framework.
+    Addresses #663 (Jon Siwek)
+
+2.0-beta-94 | 2011-12-03 15:57:19 -0800
+
+  * Adapting attribute serialization when talking to Broccoli. (Robin
+    Sommer)
+
+2.0-beta-92 | 2011-12-03 15:56:03 -0800
+
+  * Changes to Broxygen master script package index.  (Jon Siwek)
+
+    - Now only lists packages as those directories in the script hierarchy
+      that contain an __load__.bro file.
+
+    - Script packages (dirs with a __load__.bro file), can now include
+      a README (in reST format) that will automatically be appended
+      under the link to a specific package in the master package
+      index.
+
 2.0-beta-88 | 2011-12-02 17:00:58 -0800
 
   * Teach LogWriterAscii to use BRO_LOG_SUFFIX environemt variable.

VERSION

@@ -1 +1 @@
-2.0-beta-88
+2.0-beta-98

aux/broccoli

@@ -1 +1 @@
-Subproject commit d8f9d4698e8e02f493a669c5adcf897506671b5d
+Subproject commit 05d51b55dd5a8430151cff74f7876bd73385d5cd

aux/broctl

@@ -1 +1 @@
-Subproject commit be772bbada79b106db33fb9de5f56fa71226adc5
+Subproject commit 406c689c88997c4459eaf36bd2c65bce2e847c2d

doc/bin/group_index_generator.py

@@ -49,6 +49,7 @@ with open(group_list, 'r') as f_group_list:
                 if not os.path.exists(os.path.dirname(group_file)):
                     os.makedirs(os.path.dirname(group_file))
                 with open(group_file, 'w') as f_group_file:
+                    f_group_file.write(":orphan:\n\n")
                     title = "Package Index: %s\n" % os.path.dirname(group)
                     f_group_file.write(title);
                     for n in range(len(title)):

doc/scripts/CMakeLists.txt

@@ -73,12 +73,14 @@ macro(REST_TARGET srcDir broInput)
     elseif (${extension} STREQUAL ".bif.bro")
         set(group bifs)
     elseif (relDstDir)
-        set(pkgIndex ${relDstDir}/index)
-        set(group ${pkgIndex})
+        set(group ${relDstDir}/index)
         # add package index to master package list if not already in it
-        list(FIND MASTER_PKG_LIST ${pkgIndex} _found)
+        # and if a __load__.bro exists in the original script directory
+        list(FIND MASTER_PKG_LIST ${relDstDir} _found)
         if (_found EQUAL -1)
-            list(APPEND MASTER_PKG_LIST ${pkgIndex})
+            if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${relDstDir}/__load__.bro)
+                list(APPEND MASTER_PKG_LIST ${relDstDir})
+            endif ()
         endif ()
     else ()
         set(group "")
@@ -137,11 +139,15 @@ file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
 # policy/packages.rst file
 set(MASTER_PKG_INDEX_TEXT "")
 foreach (pkg ${MASTER_PKG_LIST})
-    # strip of the trailing /index for the link name
-    get_filename_component(lnktxt ${pkg} PATH)
-    # pretty-up the link name by removing common scripts/ prefix
-    string(REPLACE "scripts/" "" lnktxt "${lnktxt}")
-    set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n   ${lnktxt} <${pkg}>")
+    set(MASTER_PKG_INDEX_TEXT
+        "${MASTER_PKG_INDEX_TEXT}\n:doc:`${pkg} <${pkg}/index>`\n")
+    if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README)
+        file(STRINGS ${CMAKE_SOURCE_DIR}/scripts/${pkg}/README pkgreadme)
+        foreach (line ${pkgreadme})
+            set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n    ${line}")
+        endforeach ()
+        set(MASTER_PKG_INDEX_TEXT "${MASTER_PKG_INDEX_TEXT}\n")
+    endif ()
 endforeach ()
 file(WRITE ${MASTER_PACKAGE_INDEX} "${MASTER_PKG_INDEX_TEXT}")
 

doc/scripts/packages.rst

@@ -10,8 +10,3 @@ script, it supports being loaded in mass as a whole directory for convenience.
 Packages/scripts in the ``base/`` directory are all loaded by default, while
 ones in ``policy/`` provide functionality and customization options that are
 more appropriate for users to decide whether they'd like to load it or not.
-
-.. toctree::
-   :maxdepth: 1
-
-

scripts/base/frameworks/cluster/__load__.bro

@@ -28,17 +28,14 @@ redef Communication::listen_port = Cluster::nodes[Cluster::node]$p;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 @load ./nodes/manager
-@load site/local-manager
 @endif
 
 @if ( Cluster::local_node_type() == Cluster::PROXY )
 @load ./nodes/proxy
-@load site/local-proxy
 @endif
 
 @if ( Cluster::local_node_type() == Cluster::WORKER )
 @load ./nodes/worker
-@load site/local-worker
 @endif
 
 @endif

src/Attr.cc

@@ -481,7 +481,11 @@ bool Attributes::DoSerialize(SerialInfo* info) const
 	loop_over_list((*attrs), i)
 		{
 		Attr* a = (*attrs)[i];
-		SERIALIZE_OPTIONAL(a->AttrExpr())
+
+		// Broccoli doesn't support expressions.
+		Expr* e = (! info->broccoli_peer) ? a->AttrExpr() : 0;
+		SERIALIZE_OPTIONAL(e);
+
 		if ( ! SERIALIZE(char(a->Tag())) )
 			return false;
 		}

src/RemoteSerializer.cc

@@ -385,6 +385,9 @@ inline void RemoteSerializer::SetupSerialInfo(SerialInfo* info, Peer* peer)
 	     peer->phase == Peer::RUNNING )
 		info->new_cache_strategy = true;
 
+	if ( (peer->caps & Peer::BROCCOLI_PEER) )
+		info->broccoli_peer = true;
+
 	info->include_locations = false;
 	}
 
@@ -2113,6 +2116,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer)
 	if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) )
 		Log(LogInfo, "peer supports keep-in-cache; using that", peer);
 
+	if ( (peer->caps & Peer::BROCCOLI_PEER) )
+		Log(LogInfo, "peer is a Broccoli", peer);
+
 	if ( peer->logs_requested )
 		log_mgr->SendAllWritersTo(peer->id);
 
@@ -2365,6 +2371,9 @@ bool RemoteSerializer::ProcessSerialization()
 	     current_peer->phase == Peer::RUNNING )
 		info.new_cache_strategy = true;
 
+	if ( current_peer->caps & Peer::BROCCOLI_PEER )
+		info.broccoli_peer = true;
+
 	if ( ! forward_remote_state_changes )
 		ignore_accesses = true;
 

src/RemoteSerializer.h

@@ -198,6 +198,7 @@ protected:
 		static const int NO_CACHING = 2;
 		static const int PID_64BIT = 4;
 		static const int NEW_CACHE_STRATEGY = 8;
+		static const int BROCCOLI_PEER = 16;
 
 		// Constants to remember to who did something.
 		static const int NONE = 0;

src/SerialInfo.h

@@ -15,6 +15,7 @@ public:
 		pid_32bit = false;
 		include_locations = true;
 		new_cache_strategy = false;
+		broccoli_peer = false;
 		}
 
 	SerialInfo(const SerialInfo& info)
@@ -28,6 +29,7 @@ public:
 		pid_32bit = info.pid_32bit;
 		include_locations = info.include_locations;
 		new_cache_strategy = info.new_cache_strategy;
+		broccoli_peer = info.broccoli_peer;
 		}
 
 	// Parameters that control serialization.
@@ -46,6 +48,11 @@ public:
 	// If true, we support keeping objs in cache permanently.
 	bool new_cache_strategy;
 
+	// If true, we're connecting to a Broccoli. If so, serialization
+	// specifics may be adapted for functionality Broccoli does not
+	// support.
+	bool broccoli_peer;
+
 	ChunkedIO::Chunk* chunk; // chunk written right before the serialization
 
 	// Attributes set during serialization.
@@ -70,6 +77,7 @@ public:
 		print = 0;
 		pid_32bit = false;
 		new_cache_strategy = false;
+		broccoli_peer = false;
 		}
 
 	UnserialInfo(const UnserialInfo& info)
@@ -86,6 +94,7 @@ public:
 		print = info.print;
 		pid_32bit = info.pid_32bit;
 		new_cache_strategy = info.new_cache_strategy;
+		broccoli_peer = info.broccoli_peer;
 		}
 
 	// Parameters that control unserialization.
@@ -106,6 +115,11 @@ public:
 	// If true, we support keeping objs in cache permanently.
 	bool new_cache_strategy;
 
+	// If true, we're connecting to a Broccoli. If so, serialization
+	// specifics may be adapted for functionality Broccoli does not
+	// support.
+	bool broccoli_peer;
+
 	// If a global ID already exits, of these policies is used.
 	enum {
 		Keep,	// keep the old ID and ignore the new

testing/external/scripts/diff-all

@@ -1,6 +1,10 @@
 #! /usr/bin/env bash
 #
-# Runs btest-diff on $@ and fails if any fails.
+# Runs btest-diff on $@ and fails if any fails. If $@ contains globs, we expand
+# them relative to *both* the current directory and the test's baseline
+# directory so that we spot missing files. Note that you will need to quote
+# the globals in the TEST-EXEC line as otherwise they will have been expanded relative
+# to the current directory already when this scripts runs.
 
 diag=$TEST_DIAGNOSTICS
 
@@ -14,7 +18,10 @@ fi
 
 rc=0;
 
-for i in $@; do
+files_cwd=`ls $@`
+files_baseline=`cd $TEST_BASELINE && ls $@`
+
+for i in `echo $files_cwd $files_baseline | sort | uniq`; do
     if [[ "$i" != "loaded_scripts.log" && "$i" != "prof.log" ]]; then
         if ! btest-diff $i; then
            echo "" >>$diag

testing/external/scripts/perftools-adapt-paths

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+#
+# Adapts relative paths in perftools stderr output to work
+# directly from the top-level test directory.
+#
+# Returns an exit code > 0 if there's a leak.
+
+cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1
+
+grep -q "No leaks found" $1

testing/external/scripts/skel/test.skeleton

@@ -1,5 +1,5 @@
 # @TEST-EXEC: zcat $TRACES/trace.gz | bro -r - %INPUT
-# @TEST-EXEC: $SCRIPTS/diff-all *.log
+# @TEST-EXEC: $SCRIPTS/diff-all '*.log'
 
 @load testing-setup
 @load test-all-policy