Misc. doc/script/test cleanup.

- fixing some Metrics::add_data() call signatures
- slight refactors to cluster framework @if and adding a NONE NodeType for
  so local_node_type() will return that instead of just emitting an error
  when cluster mode isn't enabled
- `make restdoc` target now uses bro's bare-mode
- bro scripts generated from bifs now really only live in the build/src/base/
  directory and changed the DocSourcesList.cmake to dynamically figure out
  what bifs exist by looking in src/ instead of build/src/
- add some missing @load dependencies

doc/scripts/CMakeLists.txt

@@ -119,7 +119,7 @@ macro(REST_TARGET srcDir broInput)
         ARGS -E remove_directory .state
         # generate the reST documentation using bro
         COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
-        ARGS --doc-scripts ${broInput} || (rm -rf .state *.log *.rst && exit 1)
+        ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
         # move generated doc into a new directory tree that
         # defines the final structure of documents
         COMMAND "${CMAKE_COMMAND}"

doc/scripts/DocSourcesList.cmake

@@ -16,13 +16,13 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)
 
-rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@@ -34,7 +34,9 @@ rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
+rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -64,11 +66,11 @@ rest_target(${psd} base/protocols/http/partial-content.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/mime/base.bro)
 rest_target(${psd} base/protocols/mime/file-extract.bro)
 rest_target(${psd} base/protocols/mime/file-hash.bro)
 rest_target(${psd} base/protocols/mime/file-ident.bro)
-rest_target(${psd} base/protocols/rpc/base.bro)
+rest_target(${psd} base/protocols/mime/main.bro)
+rest_target(${psd} base/protocols/rpc/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
@@ -99,8 +101,7 @@ rest_target(${psd} policy/frameworks/metrics/http-example.bro)
 rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/base.bro)
+rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/event.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)

doc/scripts/genDocSourcesList.sh

@@ -68,12 +68,12 @@ sourcedir=${thisdir}/../..
 
 echo "$statictext" > $outfile
 
-bifs=`( cd ${sourcedir}/build/src && find . -name \*\.bro | sort )`
+bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
 
 for file in $bifs
 do
-    f=${file:2}
+    f=${file:2}.bro
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src $f)" >> $outfile
+    echo "rest_target(\${CMAKE_BINARY_DIR}/src/base $f)" >> $outfile
 done
 
 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

scripts/base/frameworks/cluster/__load__.bro

@@ -1,7 +1,7 @@
 # Load the core cluster support.
 @load ./main
 
-@if ( Cluster::node != "" )
+@if ( Cluster::is_enabled() )
 
 # Give the node being started up it's peer name.
 redef peer_description = Cluster::node;
@@ -26,17 +26,17 @@ redef peer_description = Cluster::node;
 ## Set the port that this node is supposed to listen on.
 redef Communication::listen_port_clear = Cluster::nodes[Cluster::node]$p;
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::MANAGER )
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
 @load ./nodes/manager
 @endif
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::PROXY )
+@if ( Cluster::local_node_type() == Cluster::PROXY )
 @load ./nodes/proxy
 @endif
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::WORKER )
+@if ( Cluster::local_node_type() == Cluster::WORKER )
 @load ./nodes/worker
 @endif
 
 @endif
-@endif
+@endif

scripts/base/frameworks/cluster/main.bro

@@ -10,6 +10,7 @@ export {
 	} &log;
 	
 	type NodeType: enum {
+		NONE,
 		CONTROL,
 		MANAGER,
 		PROXY,
@@ -54,8 +55,8 @@ export {
 	
 	## This function can be called at any time to determine what type of
 	## cluster node the current Bro instance is going to be acting as.
-	## :bro:id:`is_enabled` should be called first to find out if this is
+	## If :bro:id:`Cluster::is_enabled` returns false, then
-	## actually going to be a cluster node.
+	## :bro:enum:`Cluster::NONE` is returned.
 	global local_node_type: function(): NodeType;
 	
 	## This gives the value for the number of workers currently connected to,
@@ -81,7 +82,7 @@ function is_enabled(): bool
 
 function local_node_type(): NodeType
 	{
-	return nodes[node]$node_type;
+	return is_enabled() ? nodes[node]$node_type : NONE;
 	}
 	
 

scripts/base/frameworks/metrics/cluster.bro

@@ -5,6 +5,7 @@
 ##! transparently automated when running on a cluster.
 
 @load base/frameworks/cluster
+@load ./main
 
 module Metrics;
 
@@ -143,4 +144,4 @@ event Metrics::cluster_results(uid: string, id: ID, filter_name: string, data: M
 		}
 	}
 
-@endif
+@endif

scripts/base/frameworks/metrics/non-cluster.bro

@@ -1,3 +1,4 @@
+@load ./main
 
 module Metrics;
 

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -4,6 +4,9 @@
 ##! probably a safe assumption to make in most cases.  If both addresses
 ##! are remote, it will use the $src address.
 
+@load base/frameworks/notice/main
+@load base/utils/site
+
 module Notice;
 
 export {
@@ -44,4 +47,4 @@ event notice(n: Notice::Info) &priority=10
 		else if ( n?$dst && ! Site::is_local_addr(n$dst) )
 			n$remote_location = lookup_location(n$dst);
 		}
-	}
+	}

scripts/policy/frameworks/metrics/http-example.bro

@@ -23,12 +23,7 @@ event bro_init()
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if ( rec?$host )
-		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host]);
+		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host], 1);
 	if ( rec?$status_code )
-<<<<<<< HEAD
+		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], 1);
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $index=fmt("%d", rec$status_code)]);
 	}
-=======
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)]);
-	}
->>>>>>> master

scripts/policy/protocols/http/detect-sqli.bro

@@ -55,7 +55,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h]);
+		Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h], 1);
-		Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h]);
+		Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h], 1);
 		}
 	}

src/BroDoc.cc

@@ -60,7 +60,7 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 	if ( ! reST_file )
 		fprintf(stderr, "Failed to open %s\n", reST_filename.c_str());
 
-#ifdef DEBUG
+#ifdef DOCDEBUG
 	fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str());
 	fprintf(stdout, "\trelative dir: %s\n", rel.c_str());
 	fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str());

src/CMakeLists.txt

@@ -108,25 +108,26 @@ macro(BIF_TARGET bifInput)
                        COMMAND bifcl
                        ARGS ${CMAKE_CURRENT_SOURCE_DIR}/${bifInput} || (rm -f ${bifOutputs} && exit 1)
                        # in order be able to run bro from the build directory,
-                       # a copy of the generated bro script needs to be copied
+                       # the generated bro script needs to be inside a
-                       # to a directory tree named the same way it will be
+                       # a directory tree named the same way it will be
                        # referenced from an @load
                        COMMAND "${CMAKE_COMMAND}"
                        ARGS -E copy ${bifInput}.bro base/${bifInput}.bro
+                       COMMAND "${CMAKE_COMMAND}"
+                       ARGS -E remove -f ${bifInput}.bro
                        DEPENDS ${bifInput}
                        DEPENDS bifcl
                        COMMENT "[BIFCL] Processing ${bifInput}"
     )
     list(APPEND ALL_BIF_OUTPUTS ${bifOutputs})
     list(APPEND INSTALL_BIF_OUTPUTS
-         ${CMAKE_CURRENT_BINARY_DIR}/${bifInput}.bro)
+         ${CMAKE_CURRENT_BINARY_DIR}/base/${bifInput}.bro)
 endmacro(BIF_TARGET)
 
 # returns a list of output files that bifcl will produce
 # for given input file in ${outputFileVar}
 macro(GET_BIF_OUTPUT_FILES inputFile outputFileVar)
     set(${outputFileVar}
-        ${inputFile}.bro
         base/${inputFile}.bro
         ${inputFile}.func_def
         ${inputFile}.func_h